<!DOCTYPE html>












  


<html class="theme-next pisces use-motion" lang="zh-CN">
<head><meta name="generator" content="Hexo 3.8.0">
  <meta charset="UTF-8">
<meta http-equiv="X-UA-Compatible" content="IE=edge">
<meta name="viewport" content="width=device-width, initial-scale=1, maximum-scale=2">
<meta name="theme-color" content="#222">



















  
  
  
  

  
    
    
  

  
    
      
    

    
  

  

  
    
      
    

    
  

  
    
      
    

    
  

  
    
    
    <link rel="stylesheet" href="//fonts.googleapis.com/css?family=Monda:300,300italic,400,400italic,700,700italic|Roboto Slab:300,300italic,400,400italic,700,700italic|Lobster Two:300,300italic,400,400italic,700,700italic|PT Mono:300,300italic,400,400italic,700,700italic&subset=latin,latin-ext">
  






<link rel="stylesheet" href="/lib/font-awesome/css/font-awesome.min.css?v=4.7.0">

<link rel="stylesheet" href="/css/main.css?v=7.1.2">


  <link rel="apple-touch-icon" sizes="180x180" href="/images/apple-touch-icon-next.png?v=7.1.2">


  <link rel="icon" type="image/png" sizes="32x32" href="/images/favicon-32x32-next.png?v=7.1.2">


  <link rel="icon" type="image/png" sizes="16x16" href="/favicon.ico?v=7.1.2">


  <link rel="mask-icon" href="/images/logo.svg?v=7.1.2" color="#222">







<script id="hexo.configurations">
  var NexT = window.NexT || {};
  var CONFIG = {
    root: '/',
    scheme: 'Pisces',
    version: '7.1.2',
    sidebar: {"position":"left","display":"hide","offset":12,"onmobile":false,"dimmer":false},
    back2top: true,
    back2top_sidebar: false,
    fancybox: false,
    fastclick: false,
    lazyload: false,
    tabs: true,
    motion: {"enable":true,"async":true,"transition":{"post_block":"fadeIn","post_header":"slideDownIn","post_body":"slideDownIn","coll_header":"slideLeftIn","sidebar":"slideUpIn"}},
    algolia: {
      applicationID: '',
      apiKey: '',
      indexName: '',
      hits: {"per_page":10},
      labels: {"input_placeholder":"Search for Posts","hits_empty":"We didn't find any results for the search: ${query}","hits_stats":"${hits} results found in ${time} ms"}
    }
  };
</script>


  




  <meta name="description" content="1.环境说明在前面几期的课程里面讲过了 Flink 开发环境的搭建和应用的部署以及运行，今天的课程主要是讲 Flink 的客户端操作。本次讲解以实际操作为主。这次课程是基于社区的 Flink 1.7.2 版本，操作系统是 Mac 系统，浏览器是 Google Chrome 浏览器。有关开发环境的准备和集群的部署，请参考「开发环境搭建和应用的配置、部署及运行」的内容。">
<meta name="keywords" content="Flink">
<meta property="og:type" content="article">
<meta property="og:title" content="Apache Flink 零基础入门（四）：客户端操作的 5 种模式">
<meta property="og:url" content="https://www.dudefu.tk/Apache Flink 零基础入门（四）：客户端操作的 5 种模式.html">
<meta property="og:site_name" content="The Future">
<meta property="og:description" content="1.环境说明在前面几期的课程里面讲过了 Flink 开发环境的搭建和应用的部署以及运行，今天的课程主要是讲 Flink 的客户端操作。本次讲解以实际操作为主。这次课程是基于社区的 Flink 1.7.2 版本，操作系统是 Mac 系统，浏览器是 Google Chrome 浏览器。有关开发环境的准备和集群的部署，请参考「开发环境搭建和应用的配置、部署及运行」的内容。">
<meta property="og:locale" content="zh-CN">
<meta property="og:image" content="http://ww2.sinaimg.cn/large/006tNc79ly1g43v9p0kwnj30mz08r0t5.jpg">
<meta property="og:image" content="http://ww1.sinaimg.cn/large/006tNc79ly1g43vdcje93j30rm0omjzg.jpg">
<meta property="og:image" content="http://ww2.sinaimg.cn/large/006tNc79ly1g43y1g6tb0j31p40u0n5u.jpg">
<meta property="og:image" content="http://ww3.sinaimg.cn/large/006tNc79ly1g43xkkioblj327s0letfd.jpg">
<meta property="og:image" content="http://ww2.sinaimg.cn/large/006tNc79ly1g43xkxcutjj30zu0nqn56.jpg">
<meta property="og:image" content="http://ww2.sinaimg.cn/large/006tNc79ly1g43xllwey4j322j0u0tf2.jpg">
<meta property="og:image" content="http://ww1.sinaimg.cn/large/006tNc79ly1g43xlzk7kaj31q10u0114.jpg">
<meta property="og:image" content="http://ww3.sinaimg.cn/large/006tNc79ly1g43xp055w1j31pl0u0gu8.jpg">
<meta property="og:image" content="http://ww2.sinaimg.cn/large/006tNc79ly1g43xpq5tm4j327u0i20yu.jpg">
<meta property="og:image" content="http://ww3.sinaimg.cn/large/006tNc79ly1g43xq0vmmxj30zo0dyq5e.jpg">
<meta property="og:image" content="http://ww4.sinaimg.cn/large/006tNc79ly1g43xqdpk20j321p0u0gxy.jpg">
<meta property="og:image" content="http://ww4.sinaimg.cn/large/006tNc79ly1g43xqmithuj31ky0u0aps.jpg">
<meta property="og:image" content="http://ww2.sinaimg.cn/large/006tNc79ly1g451tdq3adj327o0r2n7a.jpg">
<meta property="og:image" content="http://ww3.sinaimg.cn/large/006tNc79ly1g451v32b68j327m0tc448.jpg">
<meta property="og:image" content="http://ww4.sinaimg.cn/large/006tNc79ly1g4400cnolyj327i0r2kaf.jpg">
<meta property="og:image" content="http://ww4.sinaimg.cn/large/006tNc79ly1g43xrwo7zyj327m0oegpl.jpg">
<meta property="og:image" content="http://ww3.sinaimg.cn/large/006tNc79ly1g43xsbntpaj31ru0u0jym.jpg">
<meta property="og:image" content="http://ww4.sinaimg.cn/large/006tNc79ly1g43xsyq9a5j31mf0u0aig.jpg">
<meta property="og:image" content="http://ww2.sinaimg.cn/large/006tNc79ly1g43xthqxvbj327q0quak2.jpg">
<meta property="og:image" content="http://ww2.sinaimg.cn/large/006tNc79ly1g43xttoiatj322k0u0wku.jpg">
<meta property="og:image" content="http://ww1.sinaimg.cn/large/006tNc79ly1g43xu3yxpej31yz0u0gt9.jpg">
<meta property="og:image" content="http://ww3.sinaimg.cn/large/006tNc79ly1g43xuggur1j31un0u0thf.jpg">
<meta property="og:image" content="http://ww1.sinaimg.cn/large/006tNc79ly1g43xuo2zerj31mi0u0qc0.jpg">
<meta property="og:image" content="http://ww1.sinaimg.cn/large/006tNc79ly1g43xv0pxotj31gp0u0diy.jpg">
<meta property="og:image" content="http://ww2.sinaimg.cn/large/006tNc79ly1g43xvba0s1j322l0u07as.jpg">
<meta property="og:image" content="http://ww4.sinaimg.cn/large/006tNc79ly1g43xvjdwmkj31ui0u0qa1.jpg">
<meta property="og:image" content="http://ww2.sinaimg.cn/large/006tNc79ly1g43xvya2m9j31gi0u00w2.jpg">
<meta property="og:image" content="http://ww3.sinaimg.cn/large/006tNc79ly1g43xw6madfj31qm0u011m.jpg">
<meta property="og:image" content="http://ww2.sinaimg.cn/large/006tNc79ly1g43xwefzizj31rg0u047a.jpg">
<meta property="og:image" content="http://ww1.sinaimg.cn/large/006tNc79ly1g43xwrrqbmj31gl0u0juc.jpg">
<meta property="og:image" content="http://ww4.sinaimg.cn/large/006tNc79ly1g43xwzgp99j31l20u0k27.jpg">
<meta property="og:image" content="http://ww1.sinaimg.cn/large/006tNc79ly1g43xx8ne9gj31rd0u0479.jpg">
<meta property="og:image" content="http://ww3.sinaimg.cn/large/006tNc79ly1g43xxmnxbjj31gr0u0whp.jpg">
<meta property="og:image" content="http://ww3.sinaimg.cn/large/006tNc79ly1g43xxyuawaj322n0u044z.jpg">
<meta property="og:image" content="http://ww2.sinaimg.cn/large/006tNc79ly1g43xyddablj31ry0u0thc.jpg">
<meta property="og:image" content="http://ww4.sinaimg.cn/large/006tNc79ly1g43xynse7uj320f0u0wl1.jpg">
<meta property="og:image" content="http://ww1.sinaimg.cn/large/006tNc79ly1g43xyvqfp5j31q40u0k00.jpg">
<meta property="og:image" content="http://ww4.sinaimg.cn/large/006tNc79ly1g43xz874fxj327u0gsjuc.jpg">
<meta property="og:image" content="http://ww4.sinaimg.cn/large/006tNc79ly1g43xzf9hyzj327u0p60y7.jpg">
<meta property="og:image" content="http://ww3.sinaimg.cn/large/006tNc79ly1g43xzppzxpj31yj0u0jy6.jpg">
<meta property="og:updated_time" content="2021-01-20T03:08:38.918Z">
<meta name="twitter:card" content="summary">
<meta name="twitter:title" content="Apache Flink 零基础入门（四）：客户端操作的 5 种模式">
<meta name="twitter:description" content="1.环境说明在前面几期的课程里面讲过了 Flink 开发环境的搭建和应用的部署以及运行，今天的课程主要是讲 Flink 的客户端操作。本次讲解以实际操作为主。这次课程是基于社区的 Flink 1.7.2 版本，操作系统是 Mac 系统，浏览器是 Google Chrome 浏览器。有关开发环境的准备和集群的部署，请参考「开发环境搭建和应用的配置、部署及运行」的内容。">
<meta name="twitter:image" content="http://ww2.sinaimg.cn/large/006tNc79ly1g43v9p0kwnj30mz08r0t5.jpg">





  
  
  <link rel="canonical" href="https://www.dudefu.tk/Apache Flink 零基础入门（四）：客户端操作的 5 种模式">



<script id="page.configurations">
  CONFIG.page = {
    sidebar: "",
  };
</script>

  <title>Apache Flink 零基础入门（四）：客户端操作的 5 种模式 | The Future</title>
  












  <noscript>
  <style>
  .use-motion .motion-element,
  .use-motion .brand,
  .use-motion .menu-item,
  .sidebar-inner,
  .use-motion .post-block,
  .use-motion .pagination,
  .use-motion .comments,
  .use-motion .post-header,
  .use-motion .post-body,
  .use-motion .collection-title { opacity: initial; }

  .use-motion .logo,
  .use-motion .site-title,
  .use-motion .site-subtitle {
    opacity: initial;
    top: initial;
  }

  .use-motion .logo-line-before i { left: initial; }
  .use-motion .logo-line-after i { right: initial; }
  </style>
</noscript>

</head>

<body itemscope="" itemtype="http://schema.org/WebPage" lang="zh-CN">

  
  
    
  

  <div class="container sidebar-position-left page-post-detail">
    <div class="headband"></div>

    <header id="header" class="header" itemscope="" itemtype="http://schema.org/WPHeader">
      <div class="header-inner"><div class="site-brand-wrapper">
  <div class="site-meta">
    

    <div class="custom-logo-site-title">
      <a href="/" class="brand" rel="start">
        <span class="logo-line-before"><i></i></span>
        <span class="site-title">The Future</span>
        <span class="logo-line-after"><i></i></span>
      </a>
    </div>
    
      
        <h1 class="site-subtitle" itemprop="description">Stay hungry,stay foolish.</h1>
      
    
    
  </div>

  <div class="site-nav-toggle">
    <button aria-label="切换导航栏">
      <span class="btn-bar"></span>
      <span class="btn-bar"></span>
      <span class="btn-bar"></span>
    </button>
  </div>
</div>



<nav class="site-nav">
  
    <ul id="menu" class="menu">
      
        
        
        
          
          <li class="menu-item menu-item-home">

    
    
    
      
    

    
      
    

    <a href="/" rel="section"><i class="menu-item-icon fa fa-fw fa-home"></i> <br>首页</a>

  </li>
        
        
        
          
          <li class="menu-item menu-item-archives">

    
    
    
      
    

    
      
    

    <a href="/archives/" rel="section"><i class="menu-item-icon fa fa-fw fa-archive"></i> <br>归档<span class="badge">125</span></a>

  </li>
        
        
        
          
          <li class="menu-item menu-item-categories">

    
    
    
      
    

    
      
    

    <a href="/categories" rel="section"><i class="menu-item-icon fa fa-fw fa-th"></i> <br>分类<span class="badge">15</span></a>

  </li>
        
        
        
          
          <li class="menu-item menu-item-tags">

    
    
    
      
    

    
      
    

    <a href="/tags" rel="section"><i class="menu-item-icon fa fa-fw fa-tags"></i> <br>标签<span class="badge">63</span></a>

  </li>
        
        
        
          
          <li class="menu-item menu-item-something">

    
    
    
      
    

    
      
    

    <a href="/something" rel="section"><i class="menu-item-icon fa fa-fw fa-paper-plane"></i> <br>干货</a>

  </li>
        
        
        
          
          <li class="menu-item menu-item-about">

    
    
    
      
    

    
      
    

    <a href="/about/" rel="section"><i class="menu-item-icon fa fa-fw fa-user"></i> <br>关于</a>

  </li>

      
      
        <li class="menu-item menu-item-search">
          
            <a href="javascript:;" class="popup-trigger">
          
            
              <i class="menu-item-icon fa fa-search fa-fw"></i> <br>搜索</a>
        </li>
      
    </ul>
  

  

  
    <div class="site-search">
      
  <div class="popup search-popup local-search-popup">
  <div class="local-search-header clearfix">
    <span class="search-icon">
      <i class="fa fa-search"></i>
    </span>
    <span class="popup-btn-close">
      <i class="fa fa-times-circle"></i>
    </span>
    <div class="local-search-input-wrapper">
      <input autocomplete="off" placeholder="搜索..." spellcheck="false" type="text" id="local-search-input">
    </div>
  </div>
  <div id="local-search-result"></div>
</div>



    </div>
  
</nav>



  



</div>
    </header>

    


    <main id="main" class="main">
      <div class="main-inner">
        <div class="content-wrap">
          
            

          
          <div id="content" class="content">
            

  <div id="posts" class="posts-expand">
    

  

  
  
  

  
    <div class="reading-progress-bar"></div>
  

  <article class="post post-type-normal" itemscope="" itemtype="http://schema.org/Article">
  
  
  
  <div class="post-block">
    <link itemprop="mainEntityOfPage" href="https://www.dudefu.tk/Apache Flink 零基础入门（四）：客户端操作的 5 种模式.html">

    <span hidden itemprop="author" itemscope="" itemtype="http://schema.org/Person">
      <meta itemprop="name" content="Daniel X">
      <meta itemprop="description" content="專注于大数据技術，分享干货">
      <meta itemprop="image" content="https://hexoblog-1254111960.cos.ap-guangzhou.myqcloud.com/HexoBlog-tou.jpg">
    </span>

    <span hidden itemprop="publisher" itemscope="" itemtype="http://schema.org/Organization">
      <meta itemprop="name" content="The Future">
    </span>

    
      <header class="post-header">

        
        
          <h2 class="post-title" itemprop="name headline">Apache Flink 零基础入门（四）：客户端操作的 5 种模式

              
            
          </h2>
        

        <div class="post-meta">
          <span class="post-time">

            
            
            

            
              <span class="post-meta-item-icon">
                <i class="fa fa-calendar-o"></i>
              </span>
              
                <span class="post-meta-item-text">发表于</span>
              

              
                
              

              <time title="创建时间：2021-01-20 10:02:53 / 修改时间：11:08:38" itemprop="dateCreated datePublished" datetime="2021-01-20T10:02:53+08:00">2021-01-20</time>
            

            
              

              
            
          </span>

          
            <span class="post-category">
            
              <span class="post-meta-divider">|</span>
            
              <span class="post-meta-item-icon">
                <i class="fa fa-folder-o"></i>
              </span>
              
                <span class="post-meta-item-text">分类于</span>
              
              
                <span itemprop="about" itemscope="" itemtype="http://schema.org/Thing"><a href="/categories/大数据/" itemprop="url" rel="index"><span itemprop="name">大数据</span></a></span>

                
                
              
            </span>
          

          
            
            
              
              <span class="post-comments-count">
                <span class="post-meta-divider">|</span>
                <span class="post-meta-item-icon">
                  <i class="fa fa-comment-o"></i>
                </span>
            
                <span class="post-meta-item-text">评论数：</span>
                <a href="/Apache Flink 零基础入门（四）：客户端操作的 5 种模式.html#comments" itemprop="discussionUrl">
                  <span class="post-comments-count valine-comment-count" data-xid="/Apache Flink 零基础入门（四）：客户端操作的 5 种模式.html" itemprop="commentCount"></span>
                </a>
              </span>
            
          

          
          
            <span id="/Apache Flink 零基础入门（四）：客户端操作的 5 种模式.html" class="leancloud_visitors" data-flag-title="Apache Flink 零基础入门（四）：客户端操作的 5 种模式">
              <span class="post-meta-divider">|</span>
              <span class="post-meta-item-icon">
                <i class="fa fa-eye"></i>
              </span>
              
                <span class="post-meta-item-text">阅读次数：</span>
              
                <span class="leancloud-visitors-count"></span>
            </span>
          

          

          

          

        </div>
      </header>
    

    
    
    
    <div class="post-body" itemprop="articleBody">

      
      

      
        <h1 id="1-环境说明"><a href="#1-环境说明" class="headerlink" title="1.环境说明"></a>1.环境说明</h1><p>在前面几期的课程里面讲过了 Flink 开发环境的搭建和应用的部署以及运行，今天的课程主要是讲 Flink 的客户端操作。本次讲解以实际操作为主。这次课程是基于社区的 Flink 1.7.2 版本，操作系统是 Mac 系统，浏览器是 Google Chrome 浏览器。有关开发环境的准备和集群的部署，请参考「<span class="exturl" data-url="aHR0cHM6Ly9kZXZlbG9wZXIuYWxpeXVuLmNvbS9hcnRpY2xlLzcxMjcwMj9zcG09YTJjNmguMTI4NzM2MzkuMC4wLjVkNzFmNTUzY3VPbXJ5JmFtcDtncm91cENvZGU9c2M=" title="https://developer.aliyun.com/article/712702?spm=a2c6h.12873639.0.0.5d71f553cuOmry&amp;groupCode=sc">开发环境搭建和应用的配置、部署及运行<i class="fa fa-external-link"></i></span>」的内容。</p>
<a id="more"></a>
<h1 id="2-课程概要"><a href="#2-课程概要" class="headerlink" title="2.课程概要"></a>2.课程概要</h1><p>如下图所示，Flink 提供了丰富的客户端操作来提交任务和与任务进行交互，包括 Flink 命令行，Scala Shell，SQL Client，Restful API 和 Web。Flink 首先提供的最重要的是命令行，其次是 SQL Client 用于提交 SQL 任务的运行，还有就是 Scala Shell 提交 Table API 的任务。同时，Flink 也提供了Restful 服务，用户可以通过 http 方式进行调用。此外，还有 Web 的方式可以提交任务。</p>
<p><img src="http://ww2.sinaimg.cn/large/006tNc79ly1g43v9p0kwnj30mz08r0t5.jpg" alt="flink_clients.png"></p>
<p>在 Flink 安装目录的 bin 目录下面可以看到有 flink, start-scala-shell.sh 和 sql-client.sh 等文件，这些都是客户端操作的入口。</p>
<p><img src="http://ww1.sinaimg.cn/large/006tNc79ly1g43vdcje93j30rm0omjzg.jpg" alt="flink_1_7_2.jpg"></p>
<h1 id="3-Flink-客户端操作"><a href="#3-Flink-客户端操作" class="headerlink" title="3.Flink 客户端操作"></a>3.Flink 客户端操作</h1><h2 id="3-1-Flink-命令行"><a href="#3-1-Flink-命令行" class="headerlink" title="3.1 Flink 命令行"></a>3.1 Flink 命令行</h2><p>Flink 的命令行参数很多，输入 flink - h 能看到完整的说明：</p>
<figure class="highlight plain"><table><tr><td class="gutter"><pre><span class="line">1</span><br></pre></td><td class="code"><pre><span class="line">flink-1.7.2 bin/flink -h</span><br></pre></td></tr></table></figure>
<p>如果想看某一个命令的参数，比如 Run 命令，输入：</p>
<figure class="highlight plain"><table><tr><td class="gutter"><pre><span class="line">1</span><br></pre></td><td class="code"><pre><span class="line">flink-1.7.2 bin/flink run -h</span><br></pre></td></tr></table></figure>
<p>本文主要讲解常见的一些操作，更详细的文档请参考: <span class="exturl" data-url="aHR0cHM6Ly9jaS5hcGFjaGUub3JnL3Byb2plY3RzL2ZsaW5rL2ZsaW5rLWRvY3Mtc3RhYmxlL29wcy9jbGkuaHRtbA==" title="https://ci.apache.org/projects/flink/flink-docs-stable/ops/cli.html">Flink 命令行官方文档<i class="fa fa-external-link"></i></span>。</p>
<h3 id="3-1-1-Standalone"><a href="#3-1-1-Standalone" class="headerlink" title="3.1.1 Standalone"></a>3.1.1 Standalone</h3><p>首先启动一个 Standalone 的集群：</p>
<figure class="highlight plain"><table><tr><td class="gutter"><pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br></pre></td><td class="code"><pre><span class="line">  flink-1.7.2 bin/start-cluster.sh</span><br><span class="line">Starting cluster.</span><br><span class="line">Starting standalonesession daemon on host zkb-MBP.local.</span><br><span class="line">Starting taskexecutor daemon on host zkb-MBP.local.</span><br></pre></td></tr></table></figure>
<p>打开 <span class="exturl" data-url="aHR0cDovLzEyNy4wLjAuMTo4MDgxLw==" title="http://127.0.0.1:8081/">http://127.0.0.1:8081<i class="fa fa-external-link"></i></span> 能看到 Web 界面。</p>
<h4 id="Run"><a href="#Run" class="headerlink" title="Run"></a>Run</h4><p>运行任务，以 Flink 自带的例子 TopSpeedWindowing 为例：</p>
<figure class="highlight plain"><table><tr><td class="gutter"><pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br><span class="line">6</span><br></pre></td><td class="code"><pre><span class="line">  flink-1.7.2 bin/flink run -d examples/streaming/TopSpeedWindowing.jar</span><br><span class="line">Starting execution of program</span><br><span class="line">Executing TopSpeedWindowing example with default input data set.</span><br><span class="line">Use --input to specify file input.</span><br><span class="line">Printing result to stdout. Use --output to specify output path.</span><br><span class="line">Job has been submitted with JobID 5e20cb6b0f357591171dfcca2eea09de</span><br></pre></td></tr></table></figure>
<p>运行起来后默认是 1 个并发:</p>
<p><img src="http://ww2.sinaimg.cn/large/006tNc79ly1g43y1g6tb0j31p40u0n5u.jpg" alt="flink_run_1.jpg"></p>
<p>点左侧「Task Manager」，然后点「Stdout」能看到输出日志：</p>
<p><img src="http://ww3.sinaimg.cn/large/006tNc79ly1g43xkkioblj327s0letfd.jpg" alt="flink_run_2.jpg"></p>
<p>或者查看本地 Log 目录下的 *.out 文件：</p>
<p><img src="http://ww2.sinaimg.cn/large/006tNc79ly1g43xkxcutjj30zu0nqn56.jpg" alt="flink_run_3.jpg"></p>
<h4 id="List"><a href="#List" class="headerlink" title="List"></a>List</h4><p>查看任务列表：</p>
<figure class="highlight plain"><table><tr><td class="gutter"><pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br><span class="line">6</span><br></pre></td><td class="code"><pre><span class="line">  flink-1.7.2 bin/flink list -m 127.0.0.1:8081</span><br><span class="line">Waiting for response...</span><br><span class="line">------------------ Running/Restarting Jobs -------------------</span><br><span class="line">24.03.2019 10:14:06 : 5e20cb6b0f357591171dfcca2eea09de : CarTopSpeedWindowingExample (RUNNING)</span><br><span class="line">--------------------------------------------------------------</span><br><span class="line">No scheduled jobs.</span><br></pre></td></tr></table></figure>
<h4 id="Stop"><a href="#Stop" class="headerlink" title="Stop"></a>Stop</h4><p>停止任务。通过 -m 来指定要停止的 JobManager 的主机地址和端口。</p>
<figure class="highlight plain"><table><tr><td class="gutter"><pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br><span class="line">6</span><br><span class="line">7</span><br><span class="line">8</span><br><span class="line">9</span><br><span class="line">10</span><br><span class="line">11</span><br><span class="line">12</span><br><span class="line">13</span><br><span class="line">14</span><br><span class="line">15</span><br><span class="line">16</span><br><span class="line">17</span><br><span class="line">18</span><br><span class="line">19</span><br><span class="line">20</span><br><span class="line">21</span><br><span class="line">22</span><br><span class="line">23</span><br><span class="line">24</span><br><span class="line">25</span><br><span class="line">26</span><br><span class="line">27</span><br><span class="line">28</span><br><span class="line">29</span><br><span class="line">30</span><br></pre></td><td class="code"><pre><span class="line">  flink-1.7.2 bin/flink stop -m 127.0.0.1:8081 d67420e52bd051fae2fddbaa79e046bb</span><br><span class="line">Stopping job d67420e52bd051fae2fddbaa79e046bb.</span><br><span class="line">------------------------------------------------------------</span><br><span class="line">The program finished with the following exception:</span><br><span class="line">  org.apache.flink.util.FlinkException: Could not stop the job   d67420e52bd051fae2fddbaa79e046bb.</span><br><span class="line">  at org.apache.flink.client.cli.CliFrontend.lambda$stop$5(CliFrontend.java:554)</span><br><span class="line">  at org.apache.flink.client.cli.CliFrontend.runClusterAction(CliFrontend.java:985)</span><br><span class="line">  at org.apache.flink.client.cli.CliFrontend.stop(CliFrontend.java:547)</span><br><span class="line">  at org.apache.flink.client.cli.CliFrontend.parseParameters(CliFrontend.java:1062)</span><br><span class="line">  at org.apache.flink.client.cli.CliFrontend.lambda$main$11(CliFrontend.java:1126)</span><br><span class="line">  at java.security.AccessController.doPrivileged(Native Method)</span><br><span class="line">  at javax.security.auth.Subject.doAs(Subject.java:422)</span><br><span class="line">  at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1836)</span><br><span class="line">  at org.apache.flink.runtime.security.HadoopSecurityContext.runSecured(HadoopSecurityContext.java:41)</span><br><span class="line">  at org.apache.flink.client.cli.CliFrontend.main(CliFrontend.java:1126)</span><br><span class="line">Caused by: java.util.concurrent.ExecutionException: org.apache.flink.runtime.rest.util.RestClientException: [Job termination (STOP) failed: This job is not stoppable.]</span><br><span class="line">  at java.util.concurrent.CompletableFuture.reportGet(CompletableFuture.java:357)</span><br><span class="line">  at java.util.concurrent.CompletableFuture.get(CompletableFuture.java:1915)</span><br><span class="line">  at org.apache.flink.client.program.rest.RestClusterClient.stop(RestClusterClient.java:392)</span><br><span class="line">  at org.apache.flink.client.cli.CliFrontend.lambda$stop$5(CliFrontend.java:552)</span><br><span class="line">... 9 more</span><br><span class="line">Caused by: org.apache.flink.runtime.rest.util.RestClientException: [Job termination (STOP) failed: This job is not stoppable.]</span><br><span class="line">  at org.apache.flink.runtime.rest.RestClient.parseResponse(RestClient.java:380)</span><br><span class="line">  at org.apache.flink.runtime.rest.RestClient.lambda$submitRequest$3(RestClient.java:364)</span><br><span class="line">  at java.util.concurrent.CompletableFuture.uniCompose(CompletableFuture.java:952)</span><br><span class="line">  at java.util.concurrent.CompletableFuture$UniCompose.tryFire(CompletableFuture.java:926)</span><br><span class="line">  at java.util.concurrent.CompletableFuture$Completion.run(CompletableFuture.java:442)</span><br><span class="line">  at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)</span><br><span class="line">  at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)</span><br><span class="line">  at java.lang.Thread.run(Thread.java:748)</span><br></pre></td></tr></table></figure>
<p>从日志里面能看出 Stop 命令执行失败了。一个 Job 能够被 Stop 要求所有的 Source 都是可以 Stoppable 的，即实现了 StoppableFunction 接口。</p>
<figure class="highlight plain"><table><tr><td class="gutter"><pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br><span class="line">6</span><br><span class="line">7</span><br><span class="line">8</span><br><span class="line">9</span><br><span class="line">10</span><br><span class="line">11</span><br><span class="line">12</span><br><span class="line">13</span><br></pre></td><td class="code"><pre><span class="line">/**</span><br><span class="line"> * 需要能 stoppable 的函数必须实现这个接口，例如流式任务的 source。</span><br><span class="line"> * stop() 方法在任务收到 STOP 信号的时候调用。</span><br><span class="line"> * source 在接收到这个信号后，必须停止发送新的数据且优雅的停止。</span><br><span class="line"> */</span><br><span class="line">@PublicEvolving</span><br><span class="line">public interface StoppableFunction &#123;</span><br><span class="line">    /**</span><br><span class="line">      * 停止 source。与 cancel() 不同的是，这是一个让 source 优雅停止的请求。</span><br><span class="line">     * 等待中的数据可以继续发送出去，不需要立即停止。</span><br><span class="line">      */</span><br><span class="line">    void stop();</span><br><span class="line">&#125;</span><br></pre></td></tr></table></figure>
<h4 id="Cancel"><a href="#Cancel" class="headerlink" title="Cancel"></a>Cancel</h4><p>取消任务。如果在 conf/flink-conf.yaml 里面配置了 state.savepoints.dir，会保存 Savepoint，否则不会保存 Savepoint。</p>
<figure class="highlight plain"><table><tr><td class="gutter"><pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br></pre></td><td class="code"><pre><span class="line">  flink-1.7.2 bin/flink cancel -m 127.0.0.1:8081 5e20cb6b0f357591171dfcca2eea09de</span><br><span class="line"> </span><br><span class="line">Cancelling job 5e20cb6b0f357591171dfcca2eea09de.</span><br><span class="line">Cancelled job 5e20cb6b0f357591171dfcca2eea09de.</span><br></pre></td></tr></table></figure>
<p>也可以在停止的时候显示指定 Savepoint 目录。</p>
<figure class="highlight plain"><table><tr><td class="gutter"><pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br><span class="line">6</span><br><span class="line">7</span><br><span class="line">8</span><br></pre></td><td class="code"><pre><span class="line">  flink-1.7.2 bin/flink cancel -m 127.0.0.1:8081 -s /tmp/savepoint 29da945b99dea6547c3fbafd57ed8759</span><br><span class="line"> </span><br><span class="line">Cancelling job 29da945b99dea6547c3fbafd57ed8759 with savepoint to /tmp/savepoint.</span><br><span class="line">Cancelled job 29da945b99dea6547c3fbafd57ed8759. Savepoint stored in file:/tmp/savepoint/savepoint-29da94-88299bacafb7.</span><br><span class="line"> </span><br><span class="line">  flink-1.7.2 ll /tmp/savepoint/savepoint-29da94-88299bacafb7</span><br><span class="line">total 32K</span><br><span class="line">-rw-r--r-- 1 baoniu 29K Mar 24 10:33 _metadata</span><br></pre></td></tr></table></figure>
<p>取消和停止（流作业）的区别如下：</p>
<ul>
<li>cancel() 调用，立即调用作业算子的 cancel() 方法，以尽快取消它们。如果算子在接到 cancel() 调用后没有停止，Flink 将开始定期中断算子线程的执行，直到所有算子停止为止。</li>
<li>stop() 调用，是更优雅的停止正在运行流作业的方式。stop() 仅适用于 Source 实现了 StoppableFunction 接口的作业。当用户请求停止作业时，作业的所有 Source 都将接收 stop() 方法调用。直到所有 Source 正常关闭时，作业才会正常结束。这种方式，使作业正常处理完所有作业。</li>
</ul>
<h4 id="Savepoint"><a href="#Savepoint" class="headerlink" title="Savepoint"></a>Savepoint</h4><p>触发 Savepoint。</p>
<figure class="highlight plain"><table><tr><td class="gutter"><pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br></pre></td><td class="code"><pre><span class="line">  flink-1.7.2 bin/flink savepoint -m 127.0.0.1:8081 ec53edcfaeb96b2a5dadbfbe5ff62bbb /tmp/savepoint</span><br><span class="line">Triggering savepoint for job ec53edcfaeb96b2a5dadbfbe5ff62bbb.</span><br><span class="line">Waiting for response...</span><br><span class="line">Savepoint completed. Path: file:/tmp/savepoint/savepoint-ec53ed-84b00ce500ee</span><br><span class="line">You can resume your program from this savepoint with the run command.</span><br></pre></td></tr></table></figure>
<p>说明：Savepoint 和 Checkpoint 的区别（<span class="exturl" data-url="aHR0cHM6Ly93d3cudmVydmVyaWNhLmNvbS9ibG9nL2RpZmZlcmVuY2VzLWJldHdlZW4tc2F2ZXBvaW50cy1hbmQtY2hlY2twb2ludHMtaW4tZmxpbms=" title="https://www.ververica.com/blog/differences-between-savepoints-and-checkpoints-in-flink">详见文档<i class="fa fa-external-link"></i></span>）：</p>
<ul>
<li>Checkpoint 是增量做的，每次的时间较短，数据量较小，只要在程序里面启用后会自动触发，用户无须感知；Checkpoint 是作业 failover 的时候自动使用，不需要用户指定。</li>
<li>Savepoint 是全量做的，每次的时间较长，数据量较大，需要用户主动去触发。Savepoint 一般用于程序的版本更新（<span class="exturl" data-url="aHR0cHM6Ly9jaS5hcGFjaGUub3JnL3Byb2plY3RzL2ZsaW5rL2ZsaW5rLWRvY3Mtc3RhYmxlL29wcy91cGdyYWRpbmcuaHRtbCNzdGVwLTEtdGFrZS1hLXNhdmVwb2ludC1pbi10aGUtb2xkLWZsaW5rLXZlcnNpb24=" title="https://ci.apache.org/projects/flink/flink-docs-stable/ops/upgrading.html#step-1-take-a-savepoint-in-the-old-flink-version">详见文档<i class="fa fa-external-link"></i></span>），Bug 修复，A/B Test 等场景，需要用户指定。</li>
</ul>
<p>通过 -s 参数从指定的 Savepoint 启动：</p>
<figure class="highlight plain"><table><tr><td class="gutter"><pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br></pre></td><td class="code"><pre><span class="line">  flink-1.7.2 bin/flink run -d -s /tmp/savepoint/savepoint-f049ff-24ec0d3e0dc7 ./examples/streaming/TopSpeedWindowing.jar</span><br><span class="line">Starting execution of program</span><br><span class="line">Executing TopSpeedWindowing example with default input data set.</span><br><span class="line">Use --input to specify file input.</span><br><span class="line">Printing result to stdout. Use --output to specify output path.</span><br></pre></td></tr></table></figure>
<p>查看 JobManager 的日志，能够看到类似这样的 Log：</p>
<figure class="highlight plain"><table><tr><td class="gutter"><pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br><span class="line">6</span><br></pre></td><td class="code"><pre><span class="line">2019-03-28 10:30:53,957 INFO  org.apache.flink.runtime.checkpoint.CheckpointCoordinator     </span><br><span class="line">- Starting job 790d7b98db6f6af55d04aec1d773852d from savepoint /tmp/savepoint/savepoint-f049ff-24ec0d3e0dc7 ()</span><br><span class="line">2019-03-28 10:30:53,959 INFO  org.apache.flink.runtime.checkpoint.CheckpointCoordinator    </span><br><span class="line"> - Reset the checkpoint ID of job 790d7b98db6f6af55d04aec1d773852d to 2.</span><br><span class="line">2019-03-28 10:30:53,959 INFO  org.apache.flink.runtime.checkpoint.CheckpointCoordinator     </span><br><span class="line">- Restoring job 790d7b98db6f6af55d04aec1d773852d from latest valid checkpoint: Checkpoint 1 @ 0 for 790d7b98db6f6af55d04aec1d773852d.</span><br></pre></td></tr></table></figure>
<h4 id="Modify"><a href="#Modify" class="headerlink" title="Modify"></a>Modify</h4><p>修改任务并行度。</p>
<p>为了方便演示，我们修改 conf/flink-conf.yaml 将 Task Slot 数从默认的 1 改为 4，并配置 Savepoint 目录。（Modify 参数后面接 -s 指定 Savepoint 路径当前版本可能有 Bug，提示无法识别）</p>
<figure class="highlight plain"><table><tr><td class="gutter"><pre><span class="line">1</span><br><span class="line">2</span><br></pre></td><td class="code"><pre><span class="line">taskmanager.numberOfTaskSlots: 4</span><br><span class="line">state.savepoints.dir: file:///tmp/savepoint</span><br></pre></td></tr></table></figure>
<p>修改参数后需要重启集群生效，然后再启动任务：</p>
<figure class="highlight plain"><table><tr><td class="gutter"><pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br><span class="line">6</span><br><span class="line">7</span><br><span class="line">8</span><br><span class="line">9</span><br><span class="line">10</span><br><span class="line">11</span><br><span class="line">12</span><br><span class="line">13</span><br></pre></td><td class="code"><pre><span class="line">  flink-1.7.2 bin/stop-cluster.sh &amp;&amp; bin/start-cluster.sh</span><br><span class="line">Stopping taskexecutor daemon (pid: 53139) on host zkb-MBP.local.</span><br><span class="line">Stopping standalonesession daemon (pid: 52723) on host zkb-MBP.local.</span><br><span class="line">Starting cluster.</span><br><span class="line">Starting standalonesession daemon on host zkb-MBP.local.</span><br><span class="line">Starting taskexecutor daemon on host zkb-MBP.local.</span><br><span class="line"> </span><br><span class="line">  flink-1.7.2 bin/flink run -d examples/streaming/TopSpeedWindowing.jar</span><br><span class="line">Starting execution of program</span><br><span class="line">Executing TopSpeedWindowing example with default input data set.</span><br><span class="line">Use --input to specify file input.</span><br><span class="line">Printing result to stdout. Use --output to specify output path.</span><br><span class="line">Job has been submitted with JobID 7752ea7b0e7303c780de9d86a5ded3fa</span><br></pre></td></tr></table></figure>
<p>从页面上能看到 Task Slot 变为了 4，这时候任务的默认并发度是 1。</p>
<p><img src="http://ww2.sinaimg.cn/large/006tNc79ly1g43xllwey4j322j0u0tf2.jpg" alt="standalone-modify-1.jpg"></p>
<p><img src="http://ww1.sinaimg.cn/large/006tNc79ly1g43xlzk7kaj31q10u0114.jpg" alt="standalone-modify-2.jpg"></p>
<p>通过 Modify 命令依次将并发度修改为 4 和 3，可以看到每次 Modify 命令都会触发一次 Savepoint。</p>
<figure class="highlight plain"><table><tr><td class="gutter"><pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br><span class="line">6</span><br><span class="line">7</span><br><span class="line">8</span><br><span class="line">9</span><br><span class="line">10</span><br><span class="line">11</span><br><span class="line">12</span><br><span class="line">13</span><br><span class="line">14</span><br><span class="line">15</span><br></pre></td><td class="code"><pre><span class="line">  flink-1.7.2 bin/flink modify -p 4 7752ea7b0e7303c780de9d86a5ded3fa</span><br><span class="line">Modify job 7752ea7b0e7303c780de9d86a5ded3fa.</span><br><span class="line">Rescaled job 7752ea7b0e7303c780de9d86a5ded3fa. Its new parallelism is 4.</span><br><span class="line"> </span><br><span class="line">  flink-1.7.2 ll /tmp/savepoint</span><br><span class="line">total 0</span><br><span class="line">drwxr-xr-x 3 baoniu 96 Jun 17 09:05 savepoint-7752ea-00c05b015836/</span><br><span class="line"> </span><br><span class="line">  flink-1.7.2 bin/flink modify -p 3 7752ea7b0e7303c780de9d86a5ded3fa</span><br><span class="line">Modify job 7752ea7b0e7303c780de9d86a5ded3fa.</span><br><span class="line">Rescaled job 7752ea7b0e7303c780de9d86a5ded3fa. Its new parallelism is 3.</span><br><span class="line"> </span><br><span class="line">  flink-1.7.2 ll /tmp/savepoint</span><br><span class="line">total 0</span><br><span class="line">drwxr-xr-x 3 baoniu 96 Jun 17 09:08 savepoint-7752ea-449b131b2bd4/</span><br></pre></td></tr></table></figure>
<p><img src="http://ww3.sinaimg.cn/large/006tNc79ly1g43xp055w1j31pl0u0gu8.jpg" alt="standalone-modify-3.jpg"></p>
<p>查看 JobManager 的日志，可以看到：</p>
<figure class="highlight plain"><table><tr><td class="gutter"><pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br><span class="line">6</span><br></pre></td><td class="code"><pre><span class="line">2019-06-17 09:05:11,179 INFO  org.apache.flink.runtime.checkpoint.CheckpointCoordinator     - Starting job 7752ea7b0e7303c780de9d86a5ded3fa from savepoint file:/tmp/savepoint/savepoint-790d7b-3581698f007e ()</span><br><span class="line">2019-06-17 09:05:11,182 INFO  org.apache.flink.runtime.checkpoint.CheckpointCoordinator     - Reset the checkpoint ID of job 7752ea7b0e7303c780de9d86a5ded3fa to 3.</span><br><span class="line">2019-06-17 09:05:11,182 INFO  org.apache.flink.runtime.checkpoint.CheckpointCoordinator     - Restoring job 790d7b98db6f6af55d04aec1d773852d from latest valid checkpoint: Checkpoint 2 @ 0 for 7752ea7b0e7303c780de9d86a5ded3fa.</span><br><span class="line">2019-06-17 09:05:11,184 INFO  org.apache.flink.runtime.checkpoint.CheckpointCoordinator     - No master state to restore</span><br><span class="line">2019-06-17 09:05:11,184 INFO  org.apache.flink.runtime.executiongraph.ExecutionGraph        - Job CarTopSpeedWindowingExample (7752ea7b0e7303c780de9d86a5ded3fa) switched from state RUNNING to SUSPENDING.</span><br><span class="line">org.apache.flink.util.FlinkException: Job is being rescaled.</span><br></pre></td></tr></table></figure>
<h4 id="Info"><a href="#Info" class="headerlink" title="Info"></a>Info</h4><p>Info 命令是用来查看 Flink 任务的执行计划（StreamGraph）的。</p>
<figure class="highlight plain"><table><tr><td class="gutter"><pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br></pre></td><td class="code"><pre><span class="line">  flink-1.7.2 bin/flink info examples/streaming/TopSpeedWindowing.jar</span><br><span class="line">----------------------- Execution Plan -----------------------</span><br><span class="line">&#123;&quot;nodes&quot;:[&#123;&quot;id&quot;:1,&quot;type&quot;:&quot;Source: Custom Source&quot;,&quot;pact&quot;:&quot;Data Source&quot;,&quot;contents&quot;:&quot;Source: Custom Source&quot;,&quot;parallelism&quot;:1&#125;,&#123;&quot;id&quot;:2,&quot;type&quot;:&quot;Timestamps/Watermarks&quot;,&quot;pact&quot;:&quot;Operator&quot;,&quot;contents&quot;:&quot;Timestamps/Watermarks&quot;,&quot;parallelism&quot;:1,&quot;predecessors&quot;:[&#123;&quot;id&quot;:1,&quot;ship_strategy&quot;:&quot;FORWARD&quot;,&quot;side&quot;:&quot;second&quot;&#125;]&#125;,&#123;&quot;id&quot;:4,&quot;type&quot;:&quot;Window(GlobalWindows(), DeltaTrigger, TimeEvictor, ComparableAggregator, PassThroughWindowFunction)&quot;,&quot;pact&quot;:&quot;Operator&quot;,&quot;contents&quot;:&quot;Window(GlobalWindows(), DeltaTrigger, TimeEvictor, ComparableAggregator, PassThroughWindowFunction)&quot;,&quot;parallelism&quot;:1,&quot;predecessors&quot;:[&#123;&quot;id&quot;:2,&quot;ship_strategy&quot;:&quot;HASH&quot;,&quot;side&quot;:&quot;second&quot;&#125;]&#125;,&#123;&quot;id&quot;:5,&quot;type&quot;:&quot;Sink: Print to Std. Out&quot;,&quot;pact&quot;:&quot;Data Sink&quot;,&quot;contents&quot;:&quot;Sink: Print to Std. Out&quot;,&quot;parallelism&quot;:1,&quot;predecessors&quot;:[&#123;&quot;id&quot;:4,&quot;ship_strategy&quot;:&quot;FORWARD&quot;,&quot;side&quot;:&quot;second&quot;&#125;]&#125;]&#125;</span><br><span class="line">--------------------------------------------------------------</span><br></pre></td></tr></table></figure>
<p>拷贝输出的 Json 内容，粘贴到这个网站：<span class="exturl" data-url="aHR0cDovL2ZsaW5rLmFwYWNoZS5vcmcvdmlzdWFsaXplci8=" title="http://flink.apache.org/visualizer/">http://flink.apache.org/visualizer/<i class="fa fa-external-link"></i></span></p>
<p><img src="http://ww2.sinaimg.cn/large/006tNc79ly1g43xpq5tm4j327u0i20yu.jpg" alt="visualizer.jpg"></p>
<p>可以和实际运行的物理执行计划对比：</p>
<p><img src="http://ww3.sinaimg.cn/large/006tNc79ly1g43xq0vmmxj30zo0dyq5e.jpg" alt="physical-execute-plan.jpg"></p>
<h3 id="3-1-2-Yarn-per-job"><a href="#3-1-2-Yarn-per-job" class="headerlink" title="3.1.2 Yarn per-job"></a>3.1.2 Yarn per-job</h3><h4 id="单任务-Attach-模式"><a href="#单任务-Attach-模式" class="headerlink" title="单任务 Attach 模式"></a>单任务 Attach 模式</h4><p>默认是 Attach 模式，即客户端会一直等待直到程序结束才会退出。</p>
<ul>
<li>通过 -m yarn-cluster 指定 Yarn 模式</li>
<li>Yarn 上显示名字为 Flink session cluster，这个 Batch 的 Wordcount 任务运行完会 FINISHED。</li>
<li>客户端能看到结果输出</li>
</ul>
<figure class="highlight plain"><table><tr><td class="gutter"><pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br><span class="line">6</span><br><span class="line">7</span><br><span class="line">8</span><br><span class="line">9</span><br><span class="line">10</span><br><span class="line">11</span><br><span class="line">12</span><br><span class="line">13</span><br><span class="line">14</span><br><span class="line">15</span><br><span class="line">16</span><br><span class="line">17</span><br><span class="line">18</span><br><span class="line">19</span><br><span class="line">20</span><br><span class="line">21</span><br><span class="line">22</span><br><span class="line">23</span><br><span class="line">24</span><br><span class="line">25</span><br><span class="line">26</span><br><span class="line">27</span><br><span class="line">28</span><br><span class="line">29</span><br><span class="line">30</span><br><span class="line">31</span><br><span class="line">32</span><br><span class="line">33</span><br><span class="line">34</span><br><span class="line">35</span><br><span class="line">36</span><br></pre></td><td class="code"><pre><span class="line">[admin@z17.sqa.zth /home/admin/flink/flink-1.7.2]</span><br><span class="line">$echo $HADOOP_CONF_DIR</span><br><span class="line">/etc/hadoop/conf/</span><br><span class="line"> </span><br><span class="line">[admin@z17.sqa.zth /home/admin/flink/flink-1.7.2]</span><br><span class="line">$./bin/flink run -m yarn-cluster ./examples/batch/WordCount.jar</span><br><span class="line"> </span><br><span class="line">2019-06-17 09:15:24,511 INFO  org.apache.hadoop.yarn.client.RMProxy                         - Connecting to ResourceManager at z05c05217.sqa.zth.tbsite.net/11.163.188.29:8050</span><br><span class="line">2019-06-17 09:15:24,690 INFO  org.apache.flink.yarn.cli.FlinkYarnSessionCli                 - No path for the flink jar passed. Using the location of class org.apache.flink.yarn.YarnClusterDescriptor to locate the jar</span><br><span class="line">2019-06-17 09:15:24,690 INFO  org.apache.flink.yarn.cli.FlinkYarnSessionCli                 - No path for the flink jar passed. Using the location of class org.apache.flink.yarn.YarnClusterDescriptor to locate the jar</span><br><span class="line">2019-06-17 09:15:24,907 INFO  org.apache.flink.yarn.AbstractYarnClusterDescriptor           - Cluster specification: ClusterSpecification&#123;masterMemoryMB=1024, taskManagerMemoryMB=1024, numberTaskManagers=1, slotsPerTaskManager=4&#125;</span><br><span class="line">2019-06-17 09:15:25,430 WARN  org.apache.hadoop.hdfs.shortcircuit.DomainSocketFactory       - The short-circuit local reads feature cannot be used because libhadoop cannot be loaded.</span><br><span class="line">2019-06-17 09:15:25,438 WARN  org.apache.flink.yarn.AbstractYarnClusterDescriptor           - The configuration directory (&apos;/Users/baoniu/Documents/work/tool/flink/flink-1.7.2/conf&apos;) contains both LOG4J and Logback configuration files. Please delete or rename one of them.</span><br><span class="line">2019-06-17 09:15:36,239 INFO  org.apache.flink.yarn.AbstractYarnClusterDescriptor           - Submitting application master application_1532332183347_0724</span><br><span class="line">2019-06-17 09:15:36,276 INFO  org.apache.hadoop.yarn.client.api.impl.YarnClientImpl         - Submitted application application_1532332183347_0724</span><br><span class="line">2019-06-17 09:15:36,276 INFO  org.apache.flink.yarn.AbstractYarnClusterDescriptor           - Waiting for the cluster to be allocated</span><br><span class="line">2019-06-17 09:15:36,281 INFO  org.apache.flink.yarn.AbstractYarnClusterDescriptor           - Deploying cluster, current state ACCEPTED</span><br><span class="line">2019-06-17 09:15:40,426 INFO  org.apache.flink.yarn.AbstractYarnClusterDescriptor           - YARN application has been deployed successfully.</span><br><span class="line">Starting execution of program</span><br><span class="line">Executing WordCount example with default input data set.</span><br><span class="line">Use --input to specify file input.</span><br><span class="line">Printing result to stdout. Use --output to specify output path.</span><br><span class="line">(a,5)</span><br><span class="line">(action,1)</span><br><span class="line">(after,1)</span><br><span class="line">(against,1)</span><br><span class="line">(all,2)</span><br><span class="line">... ...</span><br><span class="line">(would,2)</span><br><span class="line">(wrong,1)</span><br><span class="line">(you,1)</span><br><span class="line">Program execution finished</span><br><span class="line">Job with JobID 8bfe7568cb5c3254af30cbbd9cd5971e has finished.</span><br><span class="line">Job Runtime: 9371 ms</span><br><span class="line">Accumulator Results:</span><br><span class="line">- 2bed2c5506e9237fb85625416a1bc508 (java.util.ArrayList) [170 elements]</span><br></pre></td></tr></table></figure>
<p><img src="http://ww4.sinaimg.cn/large/006tNc79ly1g43xqdpk20j321p0u0gxy.jpg" alt="yarn-1.jpg"></p>
<p><img src="http://ww4.sinaimg.cn/large/006tNc79ly1g43xqmithuj31ky0u0aps.jpg" alt="yarn-2.jpg"></p>
<p>如果我们以 Attach 模式运行 Streaming 的任务，客户端会一直等待不退出，可以运行以下的例子试验下：</p>
<figure class="highlight plain"><table><tr><td class="gutter"><pre><span class="line">1</span><br></pre></td><td class="code"><pre><span class="line">./bin/flink run -m yarn-cluster ./examples/streaming/TopSpeedWindowing.jar</span><br></pre></td></tr></table></figure>
<h4 id="单任务-Detached-模式"><a href="#单任务-Detached-模式" class="headerlink" title="单任务 Detached 模式"></a>单任务 Detached 模式</h4><ul>
<li>由于是 Detached 模式，客户端提交完任务就退出了</li>
<li>Yarn 上显示为 Flink per-job cluster</li>
</ul>
<figure class="highlight plain"><table><tr><td class="gutter"><pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br><span class="line">6</span><br><span class="line">7</span><br><span class="line">8</span><br><span class="line">9</span><br><span class="line">10</span><br><span class="line">11</span><br><span class="line">12</span><br><span class="line">13</span><br><span class="line">14</span><br><span class="line">15</span><br><span class="line">16</span><br><span class="line">17</span><br></pre></td><td class="code"><pre><span class="line">$./bin/flink run -yd -m yarn-cluster ./examples/streaming/TopSpeedWindowing.jar</span><br><span class="line"> </span><br><span class="line">2019-06-18 09:21:59,247 INFO  org.apache.hadoop.yarn.client.RMProxy                         - Connecting to ResourceManager at z05c05217.sqa.zth.tbsite.net/11.163.188.29:8050</span><br><span class="line">2019-06-18 09:21:59,428 INFO  org.apache.flink.yarn.cli.FlinkYarnSessionCli                 - No path for the flink jar passed. Using the location of class org.apache.flink.yarn.YarnClusterDescriptor to locate the jar</span><br><span class="line">2019-06-18 09:21:59,428 INFO  org.apache.flink.yarn.cli.FlinkYarnSessionCli                 - No path for the flink jar passed. Using the location of class org.apache.flink.yarn.YarnClusterDescriptor to locate the jar</span><br><span class="line">2019-06-18 09:21:59,940 INFO  org.apache.flink.yarn.AbstractYarnClusterDescriptor           - Cluster specification: ClusterSpecification&#123;masterMemoryMB=1024, taskManagerMemoryMB=1024, numberTaskManagers=1, slotsPerTaskManager=4&#125;</span><br><span class="line">2019-06-18 09:22:00,427 WARN  org.apache.hadoop.hdfs.shortcircuit.DomainSocketFactory       - The short-circuit local reads feature cannot be used because libhadoop cannot be loaded.</span><br><span class="line">2019-06-18 09:22:00,436 WARN  org.apache.flink.yarn.AbstractYarnClusterDescriptor           - The configuration directory (&apos;/Users/baoniu/Documents/work/tool/flink/flink-1.7.2/conf&apos;) contains both LOG4J and Logback configuration files. Please delete or rename one of them.</span><br><span class="line">^@2019-06-18 09:22:12,113 INFO  org.apache.flink.yarn.AbstractYarnClusterDescriptor           - Submitting application master application_1532332183347_0729</span><br><span class="line">2019-06-18 09:22:12,151 INFO  org.apache.hadoop.yarn.client.api.impl.YarnClientImpl         - Submitted application application_1532332183347_0729</span><br><span class="line">2019-06-18 09:22:12,151 INFO  org.apache.flink.yarn.AbstractYarnClusterDescriptor           - Waiting for the cluster to be allocated</span><br><span class="line">2019-06-18 09:22:12,155 INFO  org.apache.flink.yarn.AbstractYarnClusterDescriptor           - Deploying cluster, current state ACCEPTED</span><br><span class="line">2019-06-18 09:22:16,275 INFO  org.apache.flink.yarn.AbstractYarnClusterDescriptor           - YARN application has been deployed successfully.</span><br><span class="line">2019-06-18 09:22:16,275 INFO  org.apache.flink.yarn.AbstractYarnClusterDescriptor           - The Flink YARN client has been started in detached mode. In order to stop Flink on YARN, use the following command or a YARN web interface to stop it:</span><br><span class="line">yarn application -kill application_1532332183347_0729</span><br><span class="line">Please also note that the temporary files of the YARN session in the home directory will not be removed.</span><br><span class="line">Job has been submitted with JobID e61b9945c33c300906ad50a9a11f36df</span><br></pre></td></tr></table></figure>
<p><img src="http://ww2.sinaimg.cn/large/006tNc79ly1g451tdq3adj327o0r2n7a.jpg" alt="yarn-detached-1.jpg"></p>
<p><img src="http://ww3.sinaimg.cn/large/006tNc79ly1g451v32b68j327m0tc448.jpg" alt="yarn-detached-2.jpg"></p>
<h3 id="3-1-3-Yarn-session"><a href="#3-1-3-Yarn-session" class="headerlink" title="3.1.3 Yarn session"></a>3.1.3 Yarn session</h3><h4 id="启动-Session"><a href="#启动-Session" class="headerlink" title="启动 Session"></a>启动 Session</h4><figure class="highlight plain"><table><tr><td class="gutter"><pre><span class="line">1</span><br></pre></td><td class="code"><pre><span class="line">./bin/yarn-session.sh -tm 2048 -s 3</span><br></pre></td></tr></table></figure>
<p>表示启动一个 Yarn session 集群，每个 TM 的内存是 2 G，每个 TM 有 3 个 Slot。(注意：-n 参数不生效)</p>
<figure class="highlight plain"><table><tr><td class="gutter"><pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br><span class="line">6</span><br><span class="line">7</span><br><span class="line">8</span><br><span class="line">9</span><br><span class="line">10</span><br><span class="line">11</span><br><span class="line">12</span><br><span class="line">13</span><br><span class="line">14</span><br><span class="line">15</span><br><span class="line">16</span><br><span class="line">17</span><br><span class="line">18</span><br><span class="line">19</span><br><span class="line">20</span><br><span class="line">21</span><br><span class="line">22</span><br><span class="line">23</span><br></pre></td><td class="code"><pre><span class="line">  flink-1.7.2 ./bin/yarn-session.sh -tm 2048 -s 3</span><br><span class="line">2019-06-17 09:21:50,177 INFO  org.apache.flink.configuration.GlobalConfiguration            - Loading configuration property: jobmanager.rpc.address, localhost</span><br><span class="line">2019-06-17 09:21:50,179 INFO  org.apache.flink.configuration.GlobalConfiguration            - Loading configuration property: jobmanager.rpc.port, 6123</span><br><span class="line">2019-06-17 09:21:50,179 INFO  org.apache.flink.configuration.GlobalConfiguration            - Loading configuration property: jobmanager.heap.size, 1024m</span><br><span class="line">2019-06-17 09:21:50,179 INFO  org.apache.flink.configuration.GlobalConfiguration            - Loading configuration property: taskmanager.heap.size, 1024m</span><br><span class="line">2019-06-17 09:21:50,179 INFO  org.apache.flink.configuration.GlobalConfiguration            - Loading configuration property: taskmanager.numberOfTaskSlots, 4</span><br><span class="line">2019-06-17 09:21:50,179 INFO  org.apache.flink.configuration.GlobalConfiguration            - Loading configuration property: state.savepoints.dir, file:///tmp/savepoint</span><br><span class="line">2019-06-17 09:21:50,180 INFO  org.apache.flink.configuration.GlobalConfiguration            - Loading configuration property: parallelism.default, 1</span><br><span class="line">2019-06-17 09:21:50,180 INFO  org.apache.flink.configuration.GlobalConfiguration            - Loading configuration property: rest.port, 8081</span><br><span class="line">2019-06-17 09:21:50,644 WARN  org.apache.hadoop.util.NativeCodeLoader                       - Unable to load native-hadoop library for your platform... using builtin-java classes where applicable</span><br><span class="line">2019-06-17 09:21:50,746 INFO  org.apache.flink.runtime.security.modules.HadoopModule        - Hadoop user set to baoniu (auth:SIMPLE)</span><br><span class="line">2019-06-17 09:21:50,848 INFO  org.apache.hadoop.yarn.client.RMProxy                         - Connecting to ResourceManager at z05c05217.sqa.zth.tbsite.net/11.163.188.29:8050</span><br><span class="line">2019-06-17 09:21:51,148 INFO  org.apache.flink.yarn.AbstractYarnClusterDescriptor           - Cluster specification: ClusterSpecification&#123;masterMemoryMB=1024, taskManagerMemoryMB=2048, numberTaskManagers=1, slotsPerTaskManager=3&#125;</span><br><span class="line">2019-06-17 09:21:51,588 WARN  org.apache.hadoop.hdfs.shortcircuit.DomainSocketFactory       - The short-circuit local reads feature cannot be used because libhadoop cannot be loaded.</span><br><span class="line">2019-06-17 09:21:51,596 WARN  org.apache.flink.yarn.AbstractYarnClusterDescriptor           - The configuration directory (&apos;/Users/baoniu/Documents/work/tool/flink/flink-1.7.2/conf&apos;) contains both LOG4J and Logback configuration files. Please delete or rename one of them.</span><br><span class="line">^@2019-06-17 09:22:03,304 INFO  org.apache.flink.yarn.AbstractYarnClusterDescriptor           - Submitting application master application_1532332183347_0726</span><br><span class="line">2019-06-17 09:22:03,336 INFO  org.apache.hadoop.yarn.client.api.impl.YarnClientImpl         - Submitted application application_1532332183347_0726</span><br><span class="line">2019-06-17 09:22:03,336 INFO  org.apache.flink.yarn.AbstractYarnClusterDescriptor           - Waiting for the cluster to be allocated</span><br><span class="line">2019-06-17 09:22:03,340 INFO  org.apache.flink.yarn.AbstractYarnClusterDescriptor           - Deploying cluster, current state ACCEPTED</span><br><span class="line">2019-06-17 09:22:07,722 INFO  org.apache.flink.yarn.AbstractYarnClusterDescriptor           - YARN application has been deployed successfully.</span><br><span class="line">2019-06-17 09:22:08,050 INFO  org.apache.flink.runtime.rest.RestClient                      - Rest client endpoint started.</span><br><span class="line">Flink JobManager is now running on z07.sqa.net:37109 with leader id 00000000-0000-0000-0000-000000000000.</span><br><span class="line">JobManager Web Interface: http://z07.sqa.net:37109</span><br></pre></td></tr></table></figure>
<p>客户端默认是 Attach 模式，不会退出：</p>
<ul>
<li>可以 ctrl + c 退出，然后再通过 ./bin/yarn-session.sh -id application_1532332183347_0726 连上来；</li>
<li>或者启动的时候用 -d 则为 detached 模式<br>Yarn 上显示为 Flink session cluster；</li>
</ul>
<p><img src="http://ww4.sinaimg.cn/large/006tNc79ly1g4400cnolyj327i0r2kaf.jpg" alt="yarn-session-1.jpg"></p>
<p><img src="http://ww4.sinaimg.cn/large/006tNc79ly1g43xrwo7zyj327m0oegpl.jpg" alt="yarn-session-2.jpg"></p>
<ul>
<li>在本机的临时目录（有些机器是 /tmp 目录）下会生成一个文件：</li>
</ul>
<figure class="highlight plain"><table><tr><td class="gutter"><pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br><span class="line">6</span><br></pre></td><td class="code"><pre><span class="line">  flink-1.7.2 cat /var/folders/2b/r6d49pcs23z43b8fqsyz885c0000gn/T/.yarn-properties-baoniu</span><br><span class="line">#Generated YARN properties file</span><br><span class="line">#Mon Jun 17 09:22:08 CST 2019</span><br><span class="line">parallelism=3</span><br><span class="line">dynamicPropertiesString=</span><br><span class="line">applicationID=application_1532332183347_0726</span><br></pre></td></tr></table></figure>
<h4 id="提交任务"><a href="#提交任务" class="headerlink" title="提交任务"></a>提交任务</h4><figure class="highlight plain"><table><tr><td class="gutter"><pre><span class="line">1</span><br></pre></td><td class="code"><pre><span class="line">./bin/flink run ./examples/batch/WordCount.jar</span><br></pre></td></tr></table></figure>
<p>将会根据 /tmp/.yarn-properties-admin 文件内容提交到了刚启动的 Session。</p>
<figure class="highlight plain"><table><tr><td class="gutter"><pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br><span class="line">6</span><br><span class="line">7</span><br><span class="line">8</span><br><span class="line">9</span><br><span class="line">10</span><br><span class="line">11</span><br><span class="line">12</span><br><span class="line">13</span><br><span class="line">14</span><br><span class="line">15</span><br><span class="line">16</span><br><span class="line">17</span><br><span class="line">18</span><br><span class="line">19</span><br><span class="line">20</span><br><span class="line">21</span><br><span class="line">22</span><br><span class="line">23</span><br><span class="line">24</span><br><span class="line">25</span><br><span class="line">26</span><br><span class="line">27</span><br><span class="line">28</span><br></pre></td><td class="code"><pre><span class="line">  flink-1.7.2 ./bin/flink run ./examples/batch/WordCount.jar</span><br><span class="line">2019-06-17 09:26:42,767 INFO  org.apache.flink.yarn.cli.FlinkYarnSessionCli                 - Found Yarn properties file under /var/folders/2b/r6d49pcs23z43b8fqsyz885c0000gn/T/.yarn-properties-baoniu.</span><br><span class="line">2019-06-17 09:26:42,767 INFO  org.apache.flink.yarn.cli.FlinkYarnSessionCli                 - Found Yarn properties file under /var/folders/2b/r6d49pcs23z43b8fqsyz885c0000gn/T/.yarn-properties-baoniu.</span><br><span class="line">2019-06-17 09:26:43,058 INFO  org.apache.flink.yarn.cli.FlinkYarnSessionCli                 - YARN properties set default parallelism to 3</span><br><span class="line">2019-06-17 09:26:43,058 INFO  org.apache.flink.yarn.cli.FlinkYarnSessionCli                 - YARN properties set default parallelism to 3</span><br><span class="line">YARN properties set default parallelism to 3</span><br><span class="line">2019-06-17 09:26:43,097 INFO  org.apache.hadoop.yarn.client.RMProxy                         - Connecting to ResourceManager at z05c05217.sqa.zth.tbsite.net/11.163.188.29:8050</span><br><span class="line">2019-06-17 09:26:43,229 INFO  org.apache.flink.yarn.cli.FlinkYarnSessionCli                 - No path for the flink jar passed. Using the location of class org.apache.flink.yarn.YarnClusterDescriptor to locate the jar</span><br><span class="line">2019-06-17 09:26:43,229 INFO  org.apache.flink.yarn.cli.FlinkYarnSessionCli                 - No path for the flink jar passed. Using the location of class org.apache.flink.yarn.YarnClusterDescriptor to locate the jar</span><br><span class="line">2019-06-17 09:26:43,327 INFO  org.apache.flink.yarn.AbstractYarnClusterDescriptor           - Found application JobManager host name &apos;z05c07216.sqa.zth.tbsite.net&apos; and port &apos;37109&apos; from supplied application id &apos;application_1532332183347_0726&apos;</span><br><span class="line">Starting execution of program</span><br><span class="line">Executing WordCount example with default input data set.</span><br><span class="line">Use --input to specify file input.</span><br><span class="line">Printing result to stdout. Use --output to specify output path.</span><br><span class="line">^@(a,5)</span><br><span class="line">(action,1)</span><br><span class="line">(after,1)</span><br><span class="line">(against,1)</span><br><span class="line">(all,2)</span><br><span class="line">(and,12)</span><br><span class="line">... ...</span><br><span class="line">(wrong,1)</span><br><span class="line">(you,1)</span><br><span class="line">Program execution finished</span><br><span class="line">Job with JobID ad9b0f1feed6d0bf6ba4e0f18b1e65ef has finished.</span><br><span class="line">Job Runtime: 9152 ms</span><br><span class="line">Accumulator Results:</span><br><span class="line">- fd07c75d503d0d9a99e4f27dd153114c (java.util.ArrayList) [170 elements]</span><br></pre></td></tr></table></figure>
<p>运行结束后 TM 的资源会释放。</p>
<p><img src="http://ww3.sinaimg.cn/large/006tNc79ly1g43xsbntpaj31ru0u0jym.jpg" alt="session-sumit-job1.jpg"></p>
<h4 id="提交到指定的-Session"><a href="#提交到指定的-Session" class="headerlink" title="提交到指定的 Session"></a>提交到指定的 Session</h4><p>通过 -yid 参数来提交到指定的 Session。</p>
<figure class="highlight plain"><table><tr><td class="gutter"><pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br><span class="line">6</span><br><span class="line">7</span><br><span class="line">8</span><br><span class="line">9</span><br><span class="line">10</span><br><span class="line">11</span><br></pre></td><td class="code"><pre><span class="line">$./bin/flink run -d -p 30 -m yarn-cluster -yid application_1532332183347_0708 ./examples/streaming/TopSpeedWindowing.jar</span><br><span class="line"> </span><br><span class="line">2019-03-24 12:36:33,668 INFO  org.apache.hadoop.yarn.client.RMProxy                         - Connecting to ResourceManager at z05c05217.sqa.zth.tbsite.net/11.163.188.29:8050</span><br><span class="line">2019-03-24 12:36:33,773 INFO  org.apache.flink.yarn.cli.FlinkYarnSessionCli                 - No path for the flink jar passed. Using the location of class org.apache.flink.yarn.YarnClusterDescriptor to locate the jar</span><br><span class="line">2019-03-24 12:36:33,773 INFO  org.apache.flink.yarn.cli.FlinkYarnSessionCli                 - No path for the flink jar passed. Using the location of class org.apache.flink.yarn.YarnClusterDescriptor to locate the jar</span><br><span class="line">2019-03-24 12:36:33,837 INFO  org.apache.flink.yarn.AbstractYarnClusterDescriptor           - Found application JobManager host name &apos;z05c05218.sqa.zth.tbsite.net&apos; and port &apos;60783&apos; from supplied application id &apos;application_1532332183347_0708&apos;</span><br><span class="line">Starting execution of program</span><br><span class="line">Executing TopSpeedWindowing example with default input data set.</span><br><span class="line">Use --input to specify file input.</span><br><span class="line">Printing result to stdout. Use --output to specify output path.</span><br><span class="line">Job has been submitted with JobID 58d5049ebbf28d515159f2f88563f5fd</span><br></pre></td></tr></table></figure>
<p><img src="http://ww4.sinaimg.cn/large/006tNc79ly1g43xsyq9a5j31mf0u0aig.jpg" alt="yarn-specifiy-session-1.jpg"></p>
<p>注：<span class="exturl" data-url="aHR0cHM6Ly9naXRodWIuY29tL2FwYWNoZS9mbGluay9jb21taXRzL2JsaW5r" title="https://github.com/apache/flink/commits/blink">Blink版本<i class="fa fa-external-link"></i></span> 的 Session 与 Flink 的 Session 的区别：</p>
<ul>
<li>Flink 的 session -n 参数不生效，而且不会提前启动 TM；</li>
<li>Blink 的 session 可以通过 -n 指定启动多少个 TM，而且 TM 会提前起来；</li>
</ul>
<h2 id="3-2-Scala-Shell"><a href="#3-2-Scala-Shell" class="headerlink" title="3.2 Scala Shell"></a>3.2 Scala Shell</h2><p>官方文档：<span class="exturl" data-url="aHR0cHM6Ly9jaS5hcGFjaGUub3JnL3Byb2plY3RzL2ZsaW5rL2ZsaW5rLWRvY3MtcmVsZWFzZS0xLjcvb3BzL3NjYWxhX3NoZWxsLmh0bWw=" title="https://ci.apache.org/projects/flink/flink-docs-release-1.7/ops/scala_shell.html">https://ci.apache.org/projects/flink/flink-docs-release-1.7/ops/scala_shell.html<i class="fa fa-external-link"></i></span></p>
<h3 id="3-2-1-Deploy"><a href="#3-2-1-Deploy" class="headerlink" title="3.2.1 Deploy"></a>3.2.1 Deploy</h3><h4 id="Local"><a href="#Local" class="headerlink" title="Local"></a>Local</h4><figure class="highlight plain"><table><tr><td class="gutter"><pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br><span class="line">6</span><br></pre></td><td class="code"><pre><span class="line">$bin/start-scala-shell.sh local</span><br><span class="line">Starting Flink Shell:</span><br><span class="line">Starting local Flink cluster (host: localhost, port: 8081).</span><br><span class="line">Connecting to Flink cluster (host: localhost, port: 8081).</span><br><span class="line">... ...</span><br><span class="line">scala&gt;</span><br></pre></td></tr></table></figure>
<p>任务运行说明：</p>
<ul>
<li>Batch 任务内置了 benv 变量，通过 print() 将结果输出到控制台；</li>
<li>Streaming 任务内置了 senv 变量，通过 senv.execute(“job name”) 来提交任务，且 Datastream 的输出只有在 Local 模式下打印到控制台；</li>
</ul>
<h4 id="Remote"><a href="#Remote" class="headerlink" title="Remote"></a>Remote</h4><p>先启动一个 yarn session cluster：</p>
<figure class="highlight plain"><table><tr><td class="gutter"><pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br><span class="line">6</span><br><span class="line">7</span><br><span class="line">8</span><br><span class="line">9</span><br><span class="line">10</span><br><span class="line">11</span><br><span class="line">12</span><br><span class="line">13</span><br></pre></td><td class="code"><pre><span class="line">$./bin/yarn-session.sh  -tm 2048 -s 3</span><br><span class="line"></span><br><span class="line">2019-03-25 09:52:16,341 INFO  org.apache.flink.configuration.GlobalConfiguration            - Loading configuration property: jobmanager.rpc.address, localhost</span><br><span class="line">2019-03-25 09:52:16,342 INFO  org.apache.flink.configuration.GlobalConfiguration            - Loading configuration property: jobmanager.rpc.port, 6123</span><br><span class="line">2019-03-25 09:52:16,342 INFO  org.apache.flink.configuration.GlobalConfiguration            - Loading configuration property: jobmanager.heap.size, 1024m</span><br><span class="line">2019-03-25 09:52:16,343 INFO  org.apache.flink.configuration.GlobalConfiguration            - Loading configuration property: taskmanager.heap.size, 1024m</span><br><span class="line">2019-03-25 09:52:16,343 INFO  org.apache.flink.configuration.GlobalConfiguration            - Loading configuration property: taskmanager.numberOfTaskSlots, 4</span><br><span class="line">2019-03-25 09:52:16,343 INFO  org.apache.flink.configuration.GlobalConfiguration            - Loading configuration property: parallelism.default, 1</span><br><span class="line">2019-03-25 09:52:16,343 INFO  org.apache.flink.configuration.GlobalConfiguration            - Loading configuration property: state.savepoints.dir, file:///tmp/savepoint</span><br><span class="line">2019-03-25 09:52:16,343 INFO  org.apache.flink.configuration.GlobalConfiguration            </span><br><span class="line">… ...</span><br><span class="line">Flink JobManager is now running on z054.sqa.net:28665 with leader id 00000000-0000-0000-0000-000000000000.</span><br><span class="line">JobManager Web Interface: http://z054.sqa.net:28665</span><br></pre></td></tr></table></figure>
<p>启动 scala shell，连到 jm：</p>
<figure class="highlight plain"><table><tr><td class="gutter"><pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br></pre></td><td class="code"><pre><span class="line">$bin/start-scala-shell.sh remote z054.sqa.net 28665</span><br><span class="line">Starting Flink Shell:</span><br><span class="line">Connecting to Flink cluster (host: z054.sqa.net, port: 28665).</span><br><span class="line">... ...</span><br><span class="line">scala&gt;</span><br></pre></td></tr></table></figure>
<h4 id="Yarn"><a href="#Yarn" class="headerlink" title="Yarn"></a>Yarn</h4><figure class="highlight plain"><table><tr><td class="gutter"><pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br><span class="line">6</span><br><span class="line">7</span><br><span class="line">8</span><br><span class="line">9</span><br><span class="line">10</span><br><span class="line">11</span><br><span class="line">12</span><br><span class="line">13</span><br><span class="line">14</span><br><span class="line">15</span><br><span class="line">16</span><br><span class="line">17</span><br><span class="line">18</span><br><span class="line">19</span><br><span class="line">20</span><br><span class="line">21</span><br><span class="line">22</span><br><span class="line">23</span><br><span class="line">24</span><br><span class="line">25</span><br><span class="line">26</span><br><span class="line">27</span><br></pre></td><td class="code"><pre><span class="line">$./bin/start-scala-shell.sh yarn -n 2 -jm 1024 -s 2 -tm 1024 -nm flink-yarn</span><br><span class="line"></span><br><span class="line">Starting Flink Shell:</span><br><span class="line">2019-03-25 09:47:44,695 INFO  org.apache.flink.configuration.GlobalConfiguration            - Loading configuration property: jobmanager.rpc.address, localhost</span><br><span class="line">2019-03-25 09:47:44,697 INFO  org.apache.flink.configuration.GlobalConfiguration            - Loading configuration property: jobmanager.rpc.port, 6123</span><br><span class="line">2019-03-25 09:47:44,697 INFO  org.apache.flink.configuration.GlobalConfiguration            - Loading configuration property: jobmanager.heap.size, 1024m</span><br><span class="line">2019-03-25 09:47:44,697 INFO  org.apache.flink.configuration.GlobalConfiguration            - Loading configuration property: taskmanager.heap.size, 1024m</span><br><span class="line">2019-03-25 09:47:44,697 INFO  org.apache.flink.configuration.GlobalConfiguration            - Loading configuration property: taskmanager.numberOfTaskSlots, 4</span><br><span class="line">2019-03-25 09:47:44,698 INFO  org.apache.flink.configuration.GlobalConfiguration            - Loading configuration property: parallelism.default, 1</span><br><span class="line">2019-03-25 09:47:44,698 INFO  org.apache.flink.configuration.GlobalConfiguration            - Loading configuration property: state.savepoints.dir, file:///tmp/savepoint</span><br><span class="line">2019-03-25 09:47:44,698 INFO  org.apache.flink.configuration.GlobalConfiguration            - Loading configuration property: rest.port, 8081</span><br><span class="line">2019-03-25 09:47:44,717 INFO  org.apache.flink.yarn.cli.FlinkYarnSessionCli                 - Found Yarn properties file under /tmp/.yarn-properties-admin.</span><br><span class="line">2019-03-25 09:47:45,041 INFO  org.apache.hadoop.yarn.client.RMProxy                         - Connecting to ResourceManager at z05c05217.sqa.zth.tbsite.net/11.163.188.29:8050</span><br><span class="line">2019-03-25 09:47:45,098 WARN  org.apache.hadoop.util.NativeCodeLoader                       - Unable to load native-hadoop library for your platform... using builtin-java classes where applicable</span><br><span class="line">2019-03-25 09:47:45,266 INFO  org.apache.flink.yarn.cli.FlinkYarnSessionCli                 - No path for the flink jar passed. Using the location of class org.apache.flink.yarn.YarnClusterDescriptor to locate the jar</span><br><span class="line">2019-03-25 09:47:45,275 INFO  org.apache.flink.yarn.cli.FlinkYarnSessionCli                 - The argument yn is deprecated in will be ignored.</span><br><span class="line">2019-03-25 09:47:45,357 INFO  org.apache.flink.yarn.AbstractYarnClusterDescriptor           - Cluster specification: ClusterSpecification&#123;masterMemoryMB=1024, taskManagerMemoryMB=1024, numberTaskManagers=2, slotsPerTaskManager=2&#125;</span><br><span class="line">2019-03-25 09:47:45,711 WARN  org.apache.hadoop.hdfs.shortcircuit.DomainSocketFactory       - The short-circuit local reads feature cannot be used because libhadoop cannot be loaded.</span><br><span class="line">2019-03-25 09:47:45,718 WARN  org.apache.flink.yarn.AbstractYarnClusterDescriptor           - The configuration directory (&apos;/home/admin/flink/flink-1.7.2/conf&apos;) contains both LOG4J and Logback configuration files. Please delete or rename one of them.</span><br><span class="line">2019-03-25 09:47:46,514 INFO  org.apache.flink.yarn.AbstractYarnClusterDescriptor           - Submitting application master application_1532332183347_0710</span><br><span class="line">2019-03-25 09:47:46,534 INFO  org.apache.hadoop.yarn.client.api.impl.YarnClientImpl         - Submitted application application_1532332183347_0710</span><br><span class="line">2019-03-25 09:47:46,534 INFO  org.apache.flink.yarn.AbstractYarnClusterDescriptor           - Waiting for the cluster to be allocated</span><br><span class="line">2019-03-25 09:47:46,535 INFO  org.apache.flink.yarn.AbstractYarnClusterDescriptor           - Deploying cluster, current state ACCEPTED</span><br><span class="line">2019-03-25 09:47:51,051 INFO  org.apache.flink.yarn.AbstractYarnClusterDescriptor           - YARN application has been deployed successfully.</span><br><span class="line">2019-03-25 09:47:51,222 INFO  org.apache.flink.runtime.rest.RestClient                      - Rest client endpoint started.</span><br><span class="line"></span><br><span class="line">Connecting to Flink cluster (host: 10.10.10.10, port: 56942).</span><br></pre></td></tr></table></figure>
<p><img src="http://ww2.sinaimg.cn/large/006tNc79ly1g43xthqxvbj327q0quak2.jpg" alt="scala-shell-yarn.jpg"></p>
<p>按 CTRL + C 退出 Shell 后，这个 Flink cluster 还会继续运行，不会退出。</p>
<h3 id="3-2-2-Execute"><a href="#3-2-2-Execute" class="headerlink" title="3.2.2 Execute"></a>3.2.2 Execute</h3><h4 id="DataSet"><a href="#DataSet" class="headerlink" title="DataSet"></a>DataSet</h4><figure class="highlight plain"><table><tr><td class="gutter"><pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br><span class="line">6</span><br><span class="line">7</span><br><span class="line">8</span><br><span class="line">9</span><br><span class="line">10</span><br><span class="line">11</span><br><span class="line">12</span><br><span class="line">13</span><br><span class="line">14</span><br><span class="line">15</span><br><span class="line">16</span><br><span class="line">17</span><br><span class="line">18</span><br><span class="line">19</span><br><span class="line">20</span><br><span class="line">21</span><br><span class="line">22</span><br><span class="line">23</span><br></pre></td><td class="code"><pre><span class="line">  flink-1.7.2 bin/stop-cluster.sh</span><br><span class="line">No taskexecutor daemon to stop on host zkb-MBP.local.</span><br><span class="line">No standalonesession daemon to stop on host zkb-MBP.local.</span><br><span class="line">  flink-1.7.2 bin/start-scala-shell.sh local</span><br><span class="line">Starting Flink Shell:</span><br><span class="line">Starting local Flink cluster (host: localhost, port: 8081).</span><br><span class="line">Connecting to Flink cluster (host: localhost, port: 8081).</span><br><span class="line"></span><br><span class="line">scala&gt; val text = benv.fromElements(&quot;To be, or not to be,--that is the question:--&quot;)</span><br><span class="line">text: org.apache.flink.api.scala.DataSet[String] = org.apache.flink.api.scala.DataSet@5b407336</span><br><span class="line"></span><br><span class="line">scala&gt; val counts = text.flatMap &#123; _.toLowerCase.split(&quot;\\W+&quot;) &#125;.map &#123; (_, 1) &#125;.groupBy(0).sum(1)</span><br><span class="line">counts: org.apache.flink.api.scala.AggregateDataSet[(String, Int)] = org.apache.flink.api.scala.AggregateDataSet@6ee34fe4</span><br><span class="line"></span><br><span class="line">scala&gt; counts.print()</span><br><span class="line">(be,2)</span><br><span class="line">(is,1)</span><br><span class="line">(not,1)</span><br><span class="line">(or,1)</span><br><span class="line">(question,1)</span><br><span class="line">(that,1)</span><br><span class="line">(the,1)</span><br><span class="line">(to,2)</span><br></pre></td></tr></table></figure>
<p>对 DataSet 任务来说，print() 会触发任务的执行。</p>
<p><img src="http://ww2.sinaimg.cn/large/006tNc79ly1g43xttoiatj322k0u0wku.jpg" alt="scala-shell-dataset-run-1.jpg"></p>
<p>也可以将结果输出到文件（先删除 /tmp/out1，不然会报错同名文件已经存在），继续执行以下命令：</p>
<figure class="highlight plain"><table><tr><td class="gutter"><pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br></pre></td><td class="code"><pre><span class="line">scala&gt; counts.writeAsText(&quot;/tmp/out1&quot;)</span><br><span class="line">res1: org.apache.flink.api.java.operators.DataSink[(String, Int)] = DataSink &apos;&lt;unnamed&gt;&apos; (TextOutputFormat (/tmp/out1) - UTF-8)</span><br><span class="line"></span><br><span class="line">scala&gt; benv.execute(&quot;batch test&quot;)</span><br><span class="line">res2: org.apache.flink.api.common.JobExecutionResult = org.apache.flink.api.common.JobExecutionResult@737652a9</span><br></pre></td></tr></table></figure>
<p>查看 /tmp/out1 文件就能看到输出结果。</p>
<figure class="highlight plain"><table><tr><td class="gutter"><pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br><span class="line">6</span><br><span class="line">7</span><br><span class="line">8</span><br><span class="line">9</span><br></pre></td><td class="code"><pre><span class="line">  flink-1.7.2 cat /tmp/out1</span><br><span class="line">(be,2)</span><br><span class="line">(is,1)</span><br><span class="line">(not,1)</span><br><span class="line">(or,1)</span><br><span class="line">(question,1)</span><br><span class="line">(that,1)</span><br><span class="line">(the,1)</span><br><span class="line">(to,2)</span><br></pre></td></tr></table></figure>
<p><img src="http://ww1.sinaimg.cn/large/006tNc79ly1g43xu3yxpej31yz0u0gt9.jpg" alt="scala-shell-dataset-run-2.jpg"></p>
<h4 id="DataSteam"><a href="#DataSteam" class="headerlink" title="DataSteam"></a>DataSteam</h4><figure class="highlight plain"><table><tr><td class="gutter"><pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br><span class="line">6</span><br><span class="line">7</span><br><span class="line">8</span><br><span class="line">9</span><br><span class="line">10</span><br><span class="line">11</span><br><span class="line">12</span><br><span class="line">13</span><br><span class="line">14</span><br><span class="line">15</span><br><span class="line">16</span><br><span class="line">17</span><br><span class="line">18</span><br><span class="line">19</span><br><span class="line">20</span><br><span class="line">21</span><br></pre></td><td class="code"><pre><span class="line">scala&gt; val textStreaming = senv.fromElements(&quot;To be, or not to be,--that is the question:--&quot;)</span><br><span class="line">textStreaming: org.apache.flink.streaming.api.scala.DataStream[String] = org.apache.flink.streaming.api.scala.DataStream@4970b93d</span><br><span class="line"></span><br><span class="line">scala&gt; val countsStreaming = textStreaming.flatMap &#123; _.toLowerCase.split(&quot;\\W+&quot;) &#125;.map &#123; (_, 1) &#125;.keyBy(0).sum(1)</span><br><span class="line">countsStreaming: org.apache.flink.streaming.api.scala.DataStream[(String, Int)] = org.apache.flink.streaming.api.scala.DataStream@6a478680</span><br><span class="line"></span><br><span class="line">scala&gt; countsStreaming.print()</span><br><span class="line">res3: org.apache.flink.streaming.api.datastream.DataStreamSink[(String, Int)] = org.apache.flink.streaming.api.datastream.DataStreamSink@42bfc11f</span><br><span class="line"></span><br><span class="line">scala&gt; senv.execute(&quot;Streaming Wordcount&quot;)</span><br><span class="line">(to,1)</span><br><span class="line">(be,1)</span><br><span class="line">(or,1)</span><br><span class="line">(not,1)</span><br><span class="line">(to,2)</span><br><span class="line">(be,2)</span><br><span class="line">(that,1)</span><br><span class="line">(is,1)</span><br><span class="line">(the,1)</span><br><span class="line">(question,1)</span><br><span class="line">res4: org.apache.flink.api.common.JobExecutionResult = org.apache.flink.api.common.JobExecutionResult@1878815a</span><br></pre></td></tr></table></figure>
<p>对 DataStream 任务，print() 并不会触发任务的执行，需要显示调用 execute(“job name”) 才会执行任务。</p>
<p><img src="http://ww3.sinaimg.cn/large/006tNc79ly1g43xuggur1j31un0u0thf.jpg" alt="scala-shell-datastream-1.jpg"></p>
<p><img src="http://ww1.sinaimg.cn/large/006tNc79ly1g43xuo2zerj31mi0u0qc0.jpg" alt="scala-shell-datastream-2.jpg"></p>
<h4 id="TableAPI"><a href="#TableAPI" class="headerlink" title="TableAPI"></a>TableAPI</h4><p>在 Blink 开源版本里面，支持了 TableAPI 方式提交任务（可以用 btenv.sqlQuery 提交 SQL 查询），社区版本 Flink 1.8 会支持 TableAPI: <span class="exturl" data-url="aHR0cHM6Ly9pc3N1ZXMuYXBhY2hlLm9yZy9qaXJhL2Jyb3dzZS9GTElOSy05NTU1" title="https://issues.apache.org/jira/browse/FLINK-9555">https://issues.apache.org/jira/browse/FLINK-9555<i class="fa fa-external-link"></i></span></p>
<h2 id="3-3-SQL-Client-Beta"><a href="#3-3-SQL-Client-Beta" class="headerlink" title="3.3 SQL Client Beta"></a>3.3 SQL Client Beta</h2><p>SQL Client 目前还只是测试版，处于开发阶段，只能用于 SQL 的原型验证，不推荐在生产环境使用。</p>
<h3 id="3-3-1-基本用法"><a href="#3-3-1-基本用法" class="headerlink" title="3.3.1 基本用法"></a>3.3.1 基本用法</h3><figure class="highlight plain"><table><tr><td class="gutter"><pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br><span class="line">6</span><br><span class="line">7</span><br><span class="line">8</span><br><span class="line">9</span><br><span class="line">10</span><br><span class="line">11</span><br><span class="line">12</span><br><span class="line">13</span><br><span class="line">14</span><br><span class="line">15</span><br><span class="line">16</span><br><span class="line">17</span><br><span class="line">18</span><br><span class="line">19</span><br><span class="line">20</span><br><span class="line">21</span><br><span class="line">22</span><br><span class="line">23</span><br><span class="line">24</span><br><span class="line">25</span><br><span class="line">26</span><br><span class="line">27</span><br><span class="line">28</span><br><span class="line">29</span><br><span class="line">30</span><br><span class="line">31</span><br><span class="line">32</span><br></pre></td><td class="code"><pre><span class="line">  flink-1.7.2 bin/start-cluster.sh</span><br><span class="line">Starting cluster.</span><br><span class="line">Starting standalonesession daemon on host zkb-MBP.local.</span><br><span class="line">Starting taskexecutor daemon on host zkb-MBP.local.</span><br><span class="line"></span><br><span class="line">  flink-1.7.2 ./bin/sql-client.sh embedded</span><br><span class="line">No default environment specified.</span><br><span class="line">Searching for &apos;/Users/baoniu/Documents/work/tool/flink/flink-1.7.2/conf/sql-client-defaults.yaml&apos;...found.</span><br><span class="line">Reading default environment from: file:/Users/baoniu/Documents/work/tool/flink/flink-1.7.2/conf/sql-client-defaults.yaml</span><br><span class="line">No session environment specified.</span><br><span class="line">Validating current environment...done.</span><br><span class="line">… …</span><br><span class="line"></span><br><span class="line">Flink SQL&gt; help;</span><br><span class="line">The following commands are available:</span><br><span class="line"></span><br><span class="line">QUIT        Quits the SQL CLI client.</span><br><span class="line">CLEAR        Clears the current terminal.</span><br><span class="line">HELP        Prints the available commands.</span><br><span class="line">SHOW TABLES        Shows all registered tables.</span><br><span class="line">SHOW FUNCTIONS        Shows all registered user-defined functions.</span><br><span class="line">DESCRIBE        Describes the schema of a table with the given name.</span><br><span class="line">EXPLAIN        Describes the execution plan of a query or table with the given name.</span><br><span class="line">SELECT        Executes a SQL SELECT query on the Flink cluster.</span><br><span class="line">INSERT INTO        Inserts the results of a SQL SELECT query into a declared table sink.</span><br><span class="line">CREATE VIEW        Creates a virtual table from a SQL query. Syntax: &apos;CREATE VIEW &lt;name&gt; AS &lt;query&gt;;&apos;</span><br><span class="line">DROP VIEW        Deletes a previously created virtual table. Syntax: &apos;DROP VIEW &lt;name&gt;;&apos;</span><br><span class="line">SOURCE        Reads a SQL SELECT query from a file and executes it on the Flink cluster.</span><br><span class="line">SET        Sets a session configuration property. Syntax: &apos;SET &lt;key&gt;=&lt;value&gt;;&apos;. Use &apos;SET;&apos; for listing all properties.</span><br><span class="line">RESET        Resets all session configuration properties.</span><br><span class="line"></span><br><span class="line">Hint: Make sure that a statement ends with &apos;;&apos; for finalizing (multi-line) statements.</span><br></pre></td></tr></table></figure>
<h4 id="Select-查询"><a href="#Select-查询" class="headerlink" title="Select 查询"></a>Select 查询</h4><figure class="highlight plain"><table><tr><td class="gutter"><pre><span class="line">1</span><br></pre></td><td class="code"><pre><span class="line">Flink SQL&gt; SELECT &apos;Hello World&apos;;</span><br></pre></td></tr></table></figure>
<p><img src="http://ww1.sinaimg.cn/large/006tNc79ly1g43xv0pxotj31gp0u0diy.jpg" alt="sql-select.jpg"></p>
<p>按 ”Q” 退出这个界面<br>打开 <span class="exturl" data-url="aHR0cDovLzEyNy4wLjAuMTo4MDgxLw==" title="http://127.0.0.1:8081/">http://127.0.0.1:8081<i class="fa fa-external-link"></i></span> 能看到这条 Select 语句产生的查询任务已经结束了。这个查询采用的是读取固定数据集的 Custom Source，输出用的是 Stream Collect Sink，且只输出一条结果。</p>
<p>注意：如果本机的临时目录存在类似 .yarn-properties-baoniu 的文件，任务会提交到 Yarn 上。</p>
<p><img src="http://ww2.sinaimg.cn/large/006tNc79ly1g43xvba0s1j322l0u07as.jpg" alt="sql-select-run-1.jpg"></p>
<p><img src="http://ww4.sinaimg.cn/large/006tNc79ly1g43xvjdwmkj31ui0u0qa1.jpg" alt="sql-select-run-2.jpg"></p>
<h4 id="Explain"><a href="#Explain" class="headerlink" title="Explain"></a>Explain</h4><p>Explain 命令可以查看 SQL 的执行计划。</p>
<figure class="highlight plain"><table><tr><td class="gutter"><pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br><span class="line">6</span><br><span class="line">7</span><br><span class="line">8</span><br><span class="line">9</span><br><span class="line">10</span><br><span class="line">11</span><br><span class="line">12</span><br><span class="line">13</span><br><span class="line">14</span><br><span class="line">15</span><br><span class="line">16</span><br><span class="line">17</span><br></pre></td><td class="code"><pre><span class="line">Flink SQL&gt; explain SELECT name, COUNT(*) AS cnt FROM (VALUES (&apos;Bob&apos;), (&apos;Alice&apos;), (&apos;Greg&apos;), (&apos;Bob&apos;)) AS NameTable(name) GROUP BY name;</span><br><span class="line"></span><br><span class="line">== Abstract Syntax Tree ==        // 抽象语法树</span><br><span class="line">LogicalAggregate(group=[&#123;0&#125;], cnt=[COUNT()])</span><br><span class="line">  LogicalValues(tuples=[[&#123; _UTF-16LE&apos;Bob  &apos; &#125;, &#123; _UTF-16LE&apos;Alice&apos; &#125;, &#123; _UTF-16LE&apos;Greg &apos; &#125;, &#123; _UTF-16LE&apos;Bob  &apos; &#125;]])</span><br><span class="line"></span><br><span class="line">== Optimized Logical Plan ==     // 优化后的逻辑执行计划</span><br><span class="line">DataStreamGroupAggregate(groupBy=[name], select=[name, COUNT(*) AS cnt])</span><br><span class="line">  DataStreamValues(tuples=[[&#123; _UTF-16LE&apos;Bob  &apos; &#125;, &#123; _UTF-16LE&apos;Alice&apos; &#125;, &#123; _UTF-16LE&apos;Greg &apos; &#125;, &#123; _UTF-16LE&apos;Bob  &apos; &#125;]])</span><br><span class="line"></span><br><span class="line">== Physical Execution Plan ==   // 物理执行计划</span><br><span class="line">Stage 3 : Data Source</span><br><span class="line">    content : collect elements with CollectionInputFormat</span><br><span class="line"></span><br><span class="line">    Stage 5 : Operator</span><br><span class="line">        content : groupBy: (name), select: (name, COUNT(*) AS cnt)</span><br><span class="line">        ship_strategy : HASH</span><br></pre></td></tr></table></figure>
<h3 id="3-3-2-结果展示"><a href="#3-3-2-结果展示" class="headerlink" title="3.3.2 结果展示"></a>3.3.2 结果展示</h3><p>SQL Client 支持两种模式来维护并展示查询结果：</p>
<ul>
<li>table mode: 在内存中物化查询结果，并以分页 table 形式展示。用户可以通过以下命令启用 table mode;</li>
</ul>
<figure class="highlight plain"><table><tr><td class="gutter"><pre><span class="line">1</span><br></pre></td><td class="code"><pre><span class="line">SET execution.result-mode=table</span><br></pre></td></tr></table></figure>
<ul>
<li>changlog mode: 不会物化查询结果，而是直接对 continuous query 产生的添加和撤回（retractions）结果进行展示。</li>
</ul>
<figure class="highlight plain"><table><tr><td class="gutter"><pre><span class="line">1</span><br></pre></td><td class="code"><pre><span class="line">SET execution.result-mode=changelog</span><br></pre></td></tr></table></figure>
<p>接下来通过实际的例子进行演示。</p>
<h4 id="Table-mode"><a href="#Table-mode" class="headerlink" title="Table mode"></a>Table mode</h4><figure class="highlight plain"><table><tr><td class="gutter"><pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br></pre></td><td class="code"><pre><span class="line">Flink SQL&gt; SET execution.result-mode=table;</span><br><span class="line">[INFO] Session property has been set.</span><br><span class="line"></span><br><span class="line">Flink SQL&gt; SELECT name, COUNT(*) AS cnt FROM (VALUES (&apos;Bob&apos;), (&apos;Alice&apos;), (&apos;Greg&apos;), (&apos;Bob&apos;)) AS NameTable(name) GROUP BY name;</span><br></pre></td></tr></table></figure>
<p>运行结果如下图所示：</p>
<p><img src="http://ww2.sinaimg.cn/large/006tNc79ly1g43xvya2m9j31gi0u00w2.jpg" alt="sql-table-mode-1.jpg"></p>
<p><img src="http://ww3.sinaimg.cn/large/006tNc79ly1g43xw6madfj31qm0u011m.jpg" alt="sql-table-mode-2.jpg"></p>
<p><img src="http://ww2.sinaimg.cn/large/006tNc79ly1g43xwefzizj31rg0u047a.jpg" alt="sql-table-mode-3.jpg"></p>
<h4 id="Changlog-mode"><a href="#Changlog-mode" class="headerlink" title="Changlog mode"></a>Changlog mode</h4><figure class="highlight plain"><table><tr><td class="gutter"><pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br></pre></td><td class="code"><pre><span class="line">Flink SQL&gt; SET execution.result-mode=changelog;</span><br><span class="line">[INFO] Session property has been set.</span><br><span class="line"></span><br><span class="line">Flink SQL&gt; SELECT name, COUNT(*) AS cnt FROM (VALUES (&apos;Bob&apos;), (&apos;Alice&apos;), (&apos;Greg&apos;), (&apos;Bob&apos;)) AS NameTable(name) GROUP BY name;</span><br></pre></td></tr></table></figure>
<p>运行结果如下图所示：</p>
<p><img src="http://ww1.sinaimg.cn/large/006tNc79ly1g43xwrrqbmj31gl0u0juc.jpg" alt="sql-change-mode-1.jpg"></p>
<p>其中 ‘-’ 代表的就是撤回消息。</p>
<p><img src="http://ww4.sinaimg.cn/large/006tNc79ly1g43xwzgp99j31l20u0k27.jpg" alt="sql-change-mode-2.jpg"></p>
<p><img src="http://ww1.sinaimg.cn/large/006tNc79ly1g43xx8ne9gj31rd0u0479.jpg" alt="sql-change-mode-3.jpg"></p>
<h3 id="3-3-3-Environment-Files"><a href="#3-3-3-Environment-Files" class="headerlink" title="3.3.3 Environment Files"></a>3.3.3 Environment Files</h3><p>目前的 SQL Client 还不支持 DDL 语句，只能通过 yaml 文件的方式来定义 SQL 查询需要的表，UDF 和运行参数等信息。</p>
<p>首先，准备 env.yaml 和 input.csv 两个文件。</p>
<figure class="highlight plain"><table><tr><td class="gutter"><pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br><span class="line">6</span><br><span class="line">7</span><br><span class="line">8</span><br><span class="line">9</span><br><span class="line">10</span><br><span class="line">11</span><br><span class="line">12</span><br><span class="line">13</span><br><span class="line">14</span><br><span class="line">15</span><br><span class="line">16</span><br><span class="line">17</span><br><span class="line">18</span><br><span class="line">19</span><br><span class="line">20</span><br><span class="line">21</span><br><span class="line">22</span><br><span class="line">23</span><br><span class="line">24</span><br><span class="line">25</span><br><span class="line">26</span><br><span class="line">27</span><br><span class="line">28</span><br><span class="line">29</span><br><span class="line">30</span><br><span class="line">31</span><br><span class="line">32</span><br><span class="line">33</span><br><span class="line">34</span><br><span class="line">35</span><br><span class="line">36</span><br><span class="line">37</span><br><span class="line">38</span><br><span class="line">39</span><br><span class="line">40</span><br><span class="line">41</span><br><span class="line">42</span><br><span class="line">43</span><br><span class="line">44</span><br><span class="line">45</span><br><span class="line">46</span><br><span class="line">47</span><br><span class="line">48</span><br><span class="line">49</span><br><span class="line">50</span><br><span class="line">51</span><br><span class="line">52</span><br><span class="line">53</span><br><span class="line">54</span><br><span class="line">55</span><br><span class="line">56</span><br><span class="line">57</span><br><span class="line">58</span><br><span class="line">59</span><br><span class="line">60</span><br><span class="line">61</span><br><span class="line">62</span><br><span class="line">63</span><br><span class="line">64</span><br><span class="line">65</span><br><span class="line">66</span><br><span class="line">67</span><br><span class="line">68</span><br><span class="line">69</span><br><span class="line">70</span><br><span class="line">71</span><br><span class="line">72</span><br></pre></td><td class="code"><pre><span class="line">  flink-1.7.2 cat /tmp/env.yaml</span><br><span class="line">tables:</span><br><span class="line">  - name: MyTableSource</span><br><span class="line">    type: source-table</span><br><span class="line">    update-mode: append</span><br><span class="line">    connector:</span><br><span class="line">      type: filesystem</span><br><span class="line">      path: &quot;/tmp/input.csv&quot;</span><br><span class="line">    format:</span><br><span class="line">      type: csv</span><br><span class="line">      fields:</span><br><span class="line">        - name: MyField1</span><br><span class="line">          type: INT</span><br><span class="line">        - name: MyField2</span><br><span class="line">          type: VARCHAR</span><br><span class="line">      line-delimiter: &quot;\n&quot;</span><br><span class="line">      comment-prefix: &quot;#&quot;</span><br><span class="line">    schema:</span><br><span class="line">      - name: MyField1</span><br><span class="line">        type: INT</span><br><span class="line">      - name: MyField2</span><br><span class="line">        type: VARCHAR</span><br><span class="line">  - name: MyCustomView</span><br><span class="line">    type: view</span><br><span class="line">    query: &quot;SELECT MyField2 FROM MyTableSource&quot;</span><br><span class="line">  - name: MyTableSink</span><br><span class="line">    type: sink-table</span><br><span class="line">    update-mode: append</span><br><span class="line">    connector:</span><br><span class="line">      type: filesystem</span><br><span class="line">      path: &quot;/tmp/output.csv&quot;</span><br><span class="line">    format:</span><br><span class="line">      type: csv</span><br><span class="line">      fields:</span><br><span class="line">        - name: MyField1</span><br><span class="line">          type: INT</span><br><span class="line">        - name: MyField2</span><br><span class="line">          type: VARCHAR</span><br><span class="line">    schema:</span><br><span class="line">      - name: MyField1</span><br><span class="line">        type: INT</span><br><span class="line">      - name: MyField2</span><br><span class="line">        type: VARCHAR</span><br><span class="line">        </span><br><span class="line"># Execution properties allow for changing the behavior of a table program.</span><br><span class="line"></span><br><span class="line">execution:</span><br><span class="line">  type: streaming                   # required: execution mode either &apos;batch&apos; or &apos;streaming&apos;</span><br><span class="line">  result-mode: table                # required: either &apos;table&apos; or &apos;changelog&apos;</span><br><span class="line">  max-table-result-rows: 1000000    # optional: maximum number of maintained rows in</span><br><span class="line">                                    #   &apos;table&apos; mode (1000000 by default, smaller 1 means unlimited)</span><br><span class="line">  time-characteristic: event-time   # optional: &apos;processing-time&apos; or &apos;event-time&apos; (default)</span><br><span class="line">  parallelism: 1                    # optional: Flink&apos;s parallelism (1 by default)</span><br><span class="line">  periodic-watermarks-interval: 200 # optional: interval for periodic watermarks (200 ms by default)</span><br><span class="line">  max-parallelism: 16               # optional: Flink&apos;s maximum parallelism (128 by default)</span><br><span class="line">  min-idle-state-retention: 0       # optional: table program&apos;s minimum idle state time</span><br><span class="line">  max-idle-state-retention: 0       # optional: table program&apos;s maximum idle state time</span><br><span class="line">  restart-strategy:                 # optional: restart strategy</span><br><span class="line">    type: fallback                  #   &quot;fallback&quot; to global restart strategy by default</span><br><span class="line"></span><br><span class="line"># Deployment properties allow for describing the cluster to which table programs are submitted to.</span><br><span class="line"></span><br><span class="line">deployment:</span><br><span class="line">  response-timeout: 5000</span><br><span class="line"></span><br><span class="line">  flink-1.7.2 cat /tmp/input.csv</span><br><span class="line">1,hello</span><br><span class="line">2,world</span><br><span class="line">3,hello world</span><br><span class="line">1,ok</span><br><span class="line">3,bye bye</span><br><span class="line">4,yes</span><br></pre></td></tr></table></figure>
<p>启动 SQL Client：</p>
<figure class="highlight plain"><table><tr><td class="gutter"><pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br><span class="line">6</span><br><span class="line">7</span><br><span class="line">8</span><br><span class="line">9</span><br><span class="line">10</span><br><span class="line">11</span><br><span class="line">12</span><br><span class="line">13</span><br><span class="line">14</span><br><span class="line">15</span><br><span class="line">16</span><br><span class="line">17</span><br><span class="line">18</span><br><span class="line">19</span><br><span class="line">20</span><br><span class="line">21</span><br><span class="line">22</span><br><span class="line">23</span><br><span class="line">24</span><br><span class="line">25</span><br><span class="line">26</span><br><span class="line">27</span><br><span class="line">28</span><br><span class="line">29</span><br><span class="line">30</span><br><span class="line">31</span><br><span class="line">32</span><br><span class="line">33</span><br><span class="line">34</span><br></pre></td><td class="code"><pre><span class="line">  flink-1.7.2 ./bin/sql-client.sh embedded -e /tmp/env.yaml</span><br><span class="line">No default environment specified.</span><br><span class="line">Searching for &apos;/Users/baoniu/Documents/work/tool/flink/flink-1.7.2/conf/sql-client-defaults.yaml&apos;...found.</span><br><span class="line">Reading default environment from: file:/Users/baoniu/Documents/work/tool/flink/flink-1.7.2/conf/sql-client-defaults.yaml</span><br><span class="line">Reading session environment from: file:/tmp/env.yaml</span><br><span class="line">Validating current environment...done.</span><br><span class="line"></span><br><span class="line">Flink SQL&gt; show tables;</span><br><span class="line">MyCustomView</span><br><span class="line">MyTableSink</span><br><span class="line">MyTableSource</span><br><span class="line"></span><br><span class="line">Flink SQL&gt; describe MyTableSource;</span><br><span class="line">root</span><br><span class="line"> |-- MyField1: Integer</span><br><span class="line"> |-- MyField2: String</span><br><span class="line"></span><br><span class="line">Flink SQL&gt; describe MyCustomView;</span><br><span class="line">root</span><br><span class="line"> |-- MyField2: String</span><br><span class="line"></span><br><span class="line">Flink SQL&gt; create view MyView1 as select MyField1 from MyTableSource;</span><br><span class="line">[INFO] View has been created.</span><br><span class="line"></span><br><span class="line">Flink SQL&gt; show tables;</span><br><span class="line">MyCustomView</span><br><span class="line">MyTableSource</span><br><span class="line">MyView1</span><br><span class="line"></span><br><span class="line">Flink SQL&gt; describe MyView1;</span><br><span class="line">root</span><br><span class="line"> |-- MyField1: Integer</span><br><span class="line"></span><br><span class="line">Flink SQL&gt; select * from MyTableSource;</span><br></pre></td></tr></table></figure>
<p><img src="http://ww3.sinaimg.cn/large/006tNc79ly1g43xxmnxbjj31gr0u0whp.jpg" alt="sql_env_file_1.jpg"></p>
<p><img src="http://ww3.sinaimg.cn/large/006tNc79ly1g43xxyuawaj322n0u044z.jpg" alt="sql_env_file_2.jpg"></p>
<p><img src="http://ww2.sinaimg.cn/large/006tNc79ly1g43xyddablj31ry0u0thc.jpg" alt="sql_env_file_3.jpg"></p>
<p>使用 insert into 写入结果表：</p>
<figure class="highlight plain"><table><tr><td class="gutter"><pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br><span class="line">6</span><br></pre></td><td class="code"><pre><span class="line">Flink SQL&gt; insert into MyTableSink select * from MyTableSource;</span><br><span class="line">[INFO] Submitting SQL update statement to the cluster...</span><br><span class="line">[INFO] Table update statement has been successfully submitted to the cluster:</span><br><span class="line">Cluster ID: StandaloneClusterId</span><br><span class="line">Job ID: 3fac2be1fd891e3e07595c684bb7b7a0</span><br><span class="line">Web interface: http://localhost:8081</span><br></pre></td></tr></table></figure>
<p><img src="http://ww4.sinaimg.cn/large/006tNc79ly1g43xynse7uj320f0u0wl1.jpg" alt="sql_insert_into_1.jpg"></p>
<p><img src="http://ww1.sinaimg.cn/large/006tNc79ly1g43xyvqfp5j31q40u0k00.jpg" alt="sql_insert_into_2.jpg"></p>
<p>查询生成的结果数据文件：</p>
<figure class="highlight plain"><table><tr><td class="gutter"><pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br><span class="line">6</span><br><span class="line">7</span><br></pre></td><td class="code"><pre><span class="line">  flink-1.7.2 cat /tmp/output.csv</span><br><span class="line">1,hello</span><br><span class="line">2,world</span><br><span class="line">3,hello world</span><br><span class="line">1,ok</span><br><span class="line">3,bye bye</span><br><span class="line">4,yes</span><br></pre></td></tr></table></figure>
<p>也可以在 Environment 文件里面定义 UDF，在 SQL Client 里面通过 「HOW FUNCTIONS」查询和使用，这里就不再说明了。</p>
<p>SQL Client 功能社区还在开发中，详见 <span class="exturl" data-url="aHR0cHM6Ly9jd2lraS5hcGFjaGUub3JnL2NvbmZsdWVuY2UvZGlzcGxheS9GTElOSy9GTElQLTI0Ky0rU1FMK0NsaWVudA==" title="https://cwiki.apache.org/confluence/display/FLINK/FLIP-24+-+SQL+Client">FLIP-24<i class="fa fa-external-link"></i></span>。</p>
<h2 id="3-4-Restful-API"><a href="#3-4-Restful-API" class="headerlink" title="3.4 Restful API"></a>3.4 Restful API</h2><p>接下来我们演示如何通过 Rest API 来提交 Jar 包和执行任务。</p>
<p>更详细的操作请参考 Flink 的 Restful API 文档：<span class="exturl" data-url="aHR0cHM6Ly9jaS5hcGFjaGUub3JnL3Byb2plY3RzL2ZsaW5rL2ZsaW5rLWRvY3Mtc3RhYmxlL21vbml0b3JpbmcvcmVzdF9hcGkuaHRtbA==" title="https://ci.apache.org/projects/flink/flink-docs-stable/monitoring/rest_api.html">https://ci.apache.org/projects/flink/flink-docs-stable/monitoring/rest_api.html<i class="fa fa-external-link"></i></span></p>
<figure class="highlight plain"><table><tr><td class="gutter"><pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br><span class="line">6</span><br><span class="line">7</span><br><span class="line">8</span><br><span class="line">9</span><br><span class="line">10</span><br><span class="line">11</span><br></pre></td><td class="code"><pre><span class="line">  flink-1.7.2 curl http://127.0.0.1:8081/overview</span><br><span class="line">&#123;&quot;taskmanagers&quot;:1,&quot;slots-total&quot;:4,&quot;slots-available&quot;:0,&quot;jobs-running&quot;:3,&quot;jobs-finished&quot;:0,&quot;jobs-cancelled&quot;:0,&quot;jobs-failed&quot;:0,&quot;flink-version&quot;:&quot;1.7.2&quot;,&quot;flink-commit&quot;:&quot;ceba8af&quot;&#125;%</span><br><span class="line"></span><br><span class="line">  flink-1.7.2 curl -X POST -H &quot;Expect:&quot; -F &quot;jarfile=@/Users/baoniu/Documents/work/tool/flink/flink-1.7.2/examples/streaming/TopSpeedWindowing.jar&quot; http://127.0.0.1:8081/jars/upload</span><br><span class="line">&#123;&quot;filename&quot;:&quot;/var/folders/2b/r6d49pcs23z43b8fqsyz885c0000gn/T/flink-web-124c4895-cf08-4eec-8e15-8263d347efc2/flink-web-upload/6077eca7-6db0-4570-a4d0-4c3e05a5dc59_TopSpeedWindowing.jar&quot;,&quot;status&quot;:&quot;success&quot;&#125;%       </span><br><span class="line">                                                                                                                                                                                                     flink-1.7.2 curl http://127.0.0.1:8081/jars</span><br><span class="line">&#123;&quot;address&quot;:&quot;http://localhost:8081&quot;,&quot;files&quot;:[&#123;&quot;id&quot;:&quot;6077eca7-6db0-4570-a4d0-4c3e05a5dc59_TopSpeedWindowing.jar&quot;,&quot;name&quot;:&quot;TopSpeedWindowing.jar&quot;,&quot;uploaded&quot;:1553743438000,&quot;entry&quot;:[&#123;&quot;name&quot;:&quot;org.apache.flink.streaming.examples.windowing.TopSpeedWindowing&quot;,&quot;description&quot;:null&#125;]&#125;]&#125;%</span><br><span class="line">                                                                                                                                         </span><br><span class="line">  flink-1.7.2 curl http://127.0.0.1:8081/jars/6077eca7-6db0-4570-a4d0-4c3e05a5dc59_TopSpeedWindowing.jar/plan</span><br><span class="line">&#123;&quot;plan&quot;:&#123;&quot;jid&quot;:&quot;41029eb3feb9132619e454ec9b2a89fb&quot;,&quot;name&quot;:&quot;CarTopSpeedWindowingExample&quot;,&quot;nodes&quot;:[&#123;&quot;id&quot;:&quot;90bea66de1c231edf33913ecd54406c1&quot;,&quot;parallelism&quot;:1,&quot;operator&quot;:&quot;&quot;,&quot;operator_strategy&quot;:&quot;&quot;,&quot;description&quot;:&quot;Window(GlobalWindows(), DeltaTrigger, TimeEvictor, ComparableAggregator, PassThroughWindowFunction) -&gt; Sink: Print to Std. Out&quot;,&quot;inputs&quot;:[&#123;&quot;num&quot;:0,&quot;id&quot;:&quot;cbc357ccb763df2852fee8c4fc7d55f2&quot;,&quot;ship_strategy&quot;:&quot;HASH&quot;,&quot;exchange&quot;:&quot;pipelined_bounded&quot;&#125;],&quot;optimizer_properties&quot;:&#123;&#125;&#125;,&#123;&quot;id&quot;:&quot;cbc357ccb763df2852fee8c4fc7d55f2&quot;,&quot;parallelism&quot;:1,&quot;operator&quot;:&quot;&quot;,&quot;operator_strategy&quot;:&quot;&quot;,&quot;description&quot;:&quot;Source: Custom Source -&gt; Timestamps/Watermarks&quot;,&quot;optimizer_properties&quot;:&#123;&#125;&#125;]&#125;&#125;%                                                                                                                                                      flink-1.7.2 curl -X POST http://127.0.0.1:8081/jars/6077eca7-6db0-4570-a4d0-4c3e05a5dc59_TopSpeedWindowing.jar/run</span><br><span class="line">&#123;&quot;jobid&quot;:&quot;04d80a24b076523d3dc5fbaa0ad5e1ad&quot;&#125;%</span><br></pre></td></tr></table></figure>
<p><img src="http://ww4.sinaimg.cn/large/006tNc79ly1g43xz874fxj327u0gsjuc.jpg" alt="rest_api_1.jpg"></p>
<p><img src="http://ww4.sinaimg.cn/large/006tNc79ly1g43xzf9hyzj327u0p60y7.jpg" alt="rest_api_2.jpg"></p>
<p>Restful API 还提供了很多监控和 Metrics 相关的功能，对于任务提交的操作也支持的比较全面。</p>
<h2 id="3-5-Web"><a href="#3-5-Web" class="headerlink" title="3.5 Web"></a>3.5 Web</h2><p>在 Flink Dashboard 页面左侧可以看到有个「Submit new Job」的地方，用户可以上传 Jar 包和显示执行计划和提交任务。Web 提交功能主要用于新手入门和演示用。</p>
<p><img src="http://ww3.sinaimg.cn/large/006tNc79ly1g43xzppzxpj31yj0u0jy6.jpg" alt="flink_web_1.jpg"></p>
<h1 id="4-结束"><a href="#4-结束" class="headerlink" title="4.结束"></a>4.结束</h1><p>本期的课程到这里就结束了，我们主要讲解了 Flink 的 5 种任务提交的方式。熟练掌握各种任务提交方式，有利于提高我们日常的开发和运维效率。</p>
<p>视频回顾：<span class="exturl" data-url="aHR0cHM6Ly96aC52ZXJ2ZXJpY2EuY29tL2RldmVsb3BlcnMvZmxpbmstdHJhaW5pbmctY291cnNlMi8=" title="https://zh.ververica.com/developers/flink-training-course2/">https://zh.ververica.com/developers/flink-training-course2/<i class="fa fa-external-link"></i></span></p>

      
    </div>

    

    
    
    

    

    
      
    
    

    

    <footer class="post-footer">
      
        
          
        
        <div class="post-tags">
          
            <a href="/tags/Flink/" rel="tag"><i class="fa fa-tag"></i> Flink</a>
          
        </div>
      

      
      
        <div class="post-widgets">
        

        

        
          
          <div class="social_share">
            
              <div>
                

<script src="//cdn.jsdelivr.net/npm/ilyabirman-likely@2/release/likely.js"></script>



<link rel="stylesheet" href="//cdn.jsdelivr.net/npm/ilyabirman-likely@2/release/likely.css">


  


<div class="likely likely-light">
	
 	 	<div class="twitter">Tweet</div>
	
 	 	<div class="facebook">Share</div>
	
 	 	<div class="linkedin">Link</div>
	
 	 	<div class="gplus">Plus</div>
	
 	 	<div class="vkontakte">Share</div>
	
 	 	<div class="odnoklassniki">Class</div>
	
 	 	<div class="telegram">Send</div>
	
 	 	<div class="whatsapp">Send</div>
	
 	 	<div class="pinterest">Pin</div>
	
</div>

              </div>
            
            
            
              <div>
                
  <div class="bdsharebuttonbox">
    <a href="#" class="bds_tsina" data-cmd="tsina" title="分享到新浪微博"></a>
    <a href="#" class="bds_douban" data-cmd="douban" title="分享到豆瓣网"></a>
    <a href="#" class="bds_sqq" data-cmd="sqq" title="分享到QQ好友"></a>
    <a href="#" class="bds_qzone" data-cmd="qzone" title="分享到QQ空间"></a>
    <a href="#" class="bds_weixin" data-cmd="weixin" title="分享到微信"></a>
    <a href="#" class="bds_tieba" data-cmd="tieba" title="分享到百度贴吧"></a>
    <a href="#" class="bds_twi" data-cmd="twi" title="分享到Twitter"></a>
    <a href="#" class="bds_fbook" data-cmd="fbook" title="分享到Facebook"></a>
    <a href="#" class="bds_more" data-cmd="more"></a>
    <a class="bds_count" data-cmd="count"></a>
  </div>
  <script>
    window._bd_share_config = {
      "common": {
        "bdText": "",
        "bdMini": "2",
        "bdMiniList": false,
        "bdPic": ""
      },
      "share": {
        "bdSize": "16",
        "bdStyle": "0"
      },
      "image": {
        "viewList": ["tsina", "douban", "sqq", "qzone", "weixin", "twi", "fbook"],
        "viewText": "分享到：",
        "viewSize": "16"
      }
    }
  </script>

<script>
  with(document)0[(getElementsByTagName('head')[0]||body).appendChild(createElement('script')).src='//bdimg.share.baidu.com/static/api/js/share.js?cdnversion='+~(-new Date()/36e5)];
</script>

              </div>
            
          </div>
        
        </div>
      
      

      
        <div class="post-nav">
          <div class="post-nav-next post-nav-item">
            
              <a href="/Apache Flink 全领域干货合集.html" rel="next" title="Apache Flink 全领域干货合集">
                <i class="fa fa-chevron-left"></i> Apache Flink 全领域干货合集
              </a>
            
          </div>

          <span class="post-nav-divider"></span>

          <div class="post-nav-prev post-nav-item">
            
              <a href="/Apache Flink 零基础入门（五）：流处理核心组件Time.html" rel="prev" title="Apache Flink 零基础入门（五）：流处理核心组件Time">
                Apache Flink 零基础入门（五）：流处理核心组件Time <i class="fa fa-chevron-right"></i>
              </a>
            
          </div>
        </div>
      

      
      
    </footer>
  </div>
  
  
  
  </article>


  </div>


          </div>
          

  
    <div class="comments" id="comments">
      <div id="lv-container" data-id="city" data-uid="MTAyMC8yOTk3My82NTM4"></div>
    </div>

  



        </div>
        
          
  
  <div class="sidebar-toggle">
    <div class="sidebar-toggle-line-wrap">
      <span class="sidebar-toggle-line sidebar-toggle-line-first"></span>
      <span class="sidebar-toggle-line sidebar-toggle-line-middle"></span>
      <span class="sidebar-toggle-line sidebar-toggle-line-last"></span>
    </div>
  </div>

  <aside id="sidebar" class="sidebar">
    <div class="sidebar-inner">

      

      
        <ul class="sidebar-nav motion-element">
          <li class="sidebar-nav-toc sidebar-nav-active" data-target="post-toc-wrap">
            文章目录
          </li>
          <li class="sidebar-nav-overview" data-target="site-overview-wrap">
            站点概览
          </li>
        </ul>
      

      <div class="site-overview-wrap sidebar-panel">
        <div class="site-overview">
          <div class="site-author motion-element" itemprop="author" itemscope="" itemtype="http://schema.org/Person">
            
              <img class="site-author-image" itemprop="image" src="https://hexoblog-1254111960.cos.ap-guangzhou.myqcloud.com/HexoBlog-tou.jpg" alt="Daniel X">
            
              <p class="site-author-name" itemprop="name">Daniel X</p>
              <div class="site-description motion-element" itemprop="description">專注于大数据技術，分享干货</div>
          </div>

          
            <nav class="site-state motion-element">
              
                <div class="site-state-item site-state-posts">
                
                  <a href="/archives/">
                
                    <span class="site-state-item-count">125</span>
                    <span class="site-state-item-name">日志</span>
                  </a>
                </div>
              

              
                
                
                <div class="site-state-item site-state-categories">
                  
                    
                      <a href="/categories">
                    
                  
                    
                    
                      
                    
                      
                    
                      
                    
                      
                    
                      
                    
                      
                    
                      
                    
                      
                    
                      
                    
                      
                    
                      
                    
                      
                    
                      
                    
                      
                    
                      
                    
                    <span class="site-state-item-count">15</span>
                    <span class="site-state-item-name">分类</span>
                  </a>
                </div>
              

              
                
                
                <div class="site-state-item site-state-tags">
                  
                    
                      <a href="/tags">
                    
                  
                    
                    
                      
                    
                      
                    
                      
                    
                      
                    
                      
                    
                      
                    
                      
                    
                      
                    
                      
                    
                      
                    
                      
                    
                      
                    
                      
                    
                      
                    
                      
                    
                      
                    
                      
                    
                      
                    
                      
                    
                      
                    
                      
                    
                      
                    
                      
                    
                      
                    
                      
                    
                      
                    
                      
                    
                      
                    
                      
                    
                      
                    
                      
                    
                      
                    
                      
                    
                      
                    
                      
                    
                      
                    
                      
                    
                      
                    
                      
                    
                      
                    
                      
                    
                      
                    
                      
                    
                      
                    
                      
                    
                      
                    
                      
                    
                      
                    
                      
                    
                      
                    
                      
                    
                      
                    
                      
                    
                      
                    
                      
                    
                      
                    
                      
                    
                      
                    
                      
                    
                      
                    
                      
                    
                      
                    
                      
                    
                    <span class="site-state-item-count">63</span>
                    <span class="site-state-item-name">标签</span>
                  </a>
                </div>
              
            </nav>
          

          

          

          
            <div class="links-of-author motion-element">
              
                <span class="links-of-author-item">
                  
                  
                    
                  
                  
                    
                  
                  <span class="exturl" data-url="aHR0cHM6Ly9naXRodWIuY29tL2R1ZGVmdQ==" title="GitHub &rarr; https://github.com/dudefu"><i class="fa fa-fw fa-github"></i>GitHub</span>
                </span>
              
                <span class="links-of-author-item">
                  
                  
                    
                  
                  
                    
                  
                  <span class="exturl" data-url="bWFpbHRvOmR1ZGVmdUBmb3htYWlsLmNvbT9zdWJqZWN0PUhlbGxvJTIwYWdhaW4=" title="E-mail &rarr; mailto:dudefu@foxmail.com?subject=Hello%20again"><i class="fa fa-fw fa-envelope"></i>E-mail</span>
                </span>
              
                <span class="links-of-author-item">
                  
                  
                    
                  
                  
                    
                  
                  <span class="exturl" data-url="aHR0cHM6Ly93ZWliby5jb20vZHVkZWZ1" title="Weibo &rarr; https://weibo.com/dudefu"><i class="fa fa-fw fa-weibo"></i>Weibo</span>
                </span>
              
                <span class="links-of-author-item">
                  
                  
                    
                  
                  
                    
                  
                  <span class="exturl" data-url="aHR0cHM6Ly93cGEucXEuY29tL21zZ3JkP3Y9MyZ1aW49MTU3NzU3MTk1OSZzaXRlPWR1ZGVmdS5pbmZvJm1lbnU9eWVz" title="QQ &rarr; https://wpa.qq.com/msgrd?v=3&uin=1577571959&site=dudefu.info&menu=yes"><i class="fa fa-fw fa-qq"></i>QQ</span>
                </span>
              
            </div>
          

          

          
          

          
            
          
          

        </div>
      </div>

      
      <!--noindex-->
        <div class="post-toc-wrap motion-element sidebar-panel sidebar-panel-active">
          <div class="post-toc">

            
            
            
            

            
              <div class="post-toc-content"><ol class="nav"><li class="nav-item nav-level-1"><a class="nav-link" href="#1-环境说明"><span class="nav-number">1.</span> <span class="nav-text">1.环境说明</span></a></li><li class="nav-item nav-level-1"><a class="nav-link" href="#2-课程概要"><span class="nav-number">2.</span> <span class="nav-text">2.课程概要</span></a></li><li class="nav-item nav-level-1"><a class="nav-link" href="#3-Flink-客户端操作"><span class="nav-number">3.</span> <span class="nav-text">3.Flink 客户端操作</span></a><ol class="nav-child"><li class="nav-item nav-level-2"><a class="nav-link" href="#3-1-Flink-命令行"><span class="nav-number">3.1.</span> <span class="nav-text">3.1 Flink 命令行</span></a><ol class="nav-child"><li class="nav-item nav-level-3"><a class="nav-link" href="#3-1-1-Standalone"><span class="nav-number">3.1.1.</span> <span class="nav-text">3.1.1 Standalone</span></a><ol class="nav-child"><li class="nav-item nav-level-4"><a class="nav-link" href="#Run"><span class="nav-number">3.1.1.1.</span> <span class="nav-text">Run</span></a></li><li class="nav-item nav-level-4"><a class="nav-link" href="#List"><span class="nav-number">3.1.1.2.</span> <span class="nav-text">List</span></a></li><li class="nav-item nav-level-4"><a class="nav-link" href="#Stop"><span class="nav-number">3.1.1.3.</span> <span class="nav-text">Stop</span></a></li><li class="nav-item nav-level-4"><a class="nav-link" href="#Cancel"><span class="nav-number">3.1.1.4.</span> <span class="nav-text">Cancel</span></a></li><li class="nav-item nav-level-4"><a class="nav-link" href="#Savepoint"><span class="nav-number">3.1.1.5.</span> <span class="nav-text">Savepoint</span></a></li><li class="nav-item nav-level-4"><a class="nav-link" href="#Modify"><span class="nav-number">3.1.1.6.</span> <span class="nav-text">Modify</span></a></li><li class="nav-item nav-level-4"><a class="nav-link" href="#Info"><span class="nav-number">3.1.1.7.</span> <span class="nav-text">Info</span></a></li></ol></li><li class="nav-item nav-level-3"><a class="nav-link" href="#3-1-2-Yarn-per-job"><span class="nav-number">3.1.2.</span> <span class="nav-text">3.1.2 Yarn per-job</span></a><ol class="nav-child"><li class="nav-item nav-level-4"><a class="nav-link" href="#单任务-Attach-模式"><span class="nav-number">3.1.2.1.</span> <span class="nav-text">单任务 Attach 模式</span></a></li><li class="nav-item nav-level-4"><a class="nav-link" href="#单任务-Detached-模式"><span class="nav-number">3.1.2.2.</span> <span class="nav-text">单任务 Detached 模式</span></a></li></ol></li><li class="nav-item nav-level-3"><a class="nav-link" href="#3-1-3-Yarn-session"><span class="nav-number">3.1.3.</span> <span class="nav-text">3.1.3 Yarn session</span></a><ol class="nav-child"><li class="nav-item nav-level-4"><a class="nav-link" href="#启动-Session"><span class="nav-number">3.1.3.1.</span> <span class="nav-text">启动 Session</span></a></li><li class="nav-item nav-level-4"><a class="nav-link" href="#提交任务"><span class="nav-number">3.1.3.2.</span> <span class="nav-text">提交任务</span></a></li><li class="nav-item nav-level-4"><a class="nav-link" href="#提交到指定的-Session"><span class="nav-number">3.1.3.3.</span> <span class="nav-text">提交到指定的 Session</span></a></li></ol></li></ol></li><li class="nav-item nav-level-2"><a class="nav-link" href="#3-2-Scala-Shell"><span class="nav-number">3.2.</span> <span class="nav-text">3.2 Scala Shell</span></a><ol class="nav-child"><li class="nav-item nav-level-3"><a class="nav-link" href="#3-2-1-Deploy"><span class="nav-number">3.2.1.</span> <span class="nav-text">3.2.1 Deploy</span></a><ol class="nav-child"><li class="nav-item nav-level-4"><a class="nav-link" href="#Local"><span class="nav-number">3.2.1.1.</span> <span class="nav-text">Local</span></a></li><li class="nav-item nav-level-4"><a class="nav-link" href="#Remote"><span class="nav-number">3.2.1.2.</span> <span class="nav-text">Remote</span></a></li><li class="nav-item nav-level-4"><a class="nav-link" href="#Yarn"><span class="nav-number">3.2.1.3.</span> <span class="nav-text">Yarn</span></a></li></ol></li><li class="nav-item nav-level-3"><a class="nav-link" href="#3-2-2-Execute"><span class="nav-number">3.2.2.</span> <span class="nav-text">3.2.2 Execute</span></a><ol class="nav-child"><li class="nav-item nav-level-4"><a class="nav-link" href="#DataSet"><span class="nav-number">3.2.2.1.</span> <span class="nav-text">DataSet</span></a></li><li class="nav-item nav-level-4"><a class="nav-link" href="#DataSteam"><span class="nav-number">3.2.2.2.</span> <span class="nav-text">DataSteam</span></a></li><li class="nav-item nav-level-4"><a class="nav-link" href="#TableAPI"><span class="nav-number">3.2.2.3.</span> <span class="nav-text">TableAPI</span></a></li></ol></li></ol></li><li class="nav-item nav-level-2"><a class="nav-link" href="#3-3-SQL-Client-Beta"><span class="nav-number">3.3.</span> <span class="nav-text">3.3 SQL Client Beta</span></a><ol class="nav-child"><li class="nav-item nav-level-3"><a class="nav-link" href="#3-3-1-基本用法"><span class="nav-number">3.3.1.</span> <span class="nav-text">3.3.1 基本用法</span></a><ol class="nav-child"><li class="nav-item nav-level-4"><a class="nav-link" href="#Select-查询"><span class="nav-number">3.3.1.1.</span> <span class="nav-text">Select 查询</span></a></li><li class="nav-item nav-level-4"><a class="nav-link" href="#Explain"><span class="nav-number">3.3.1.2.</span> <span class="nav-text">Explain</span></a></li></ol></li><li class="nav-item nav-level-3"><a class="nav-link" href="#3-3-2-结果展示"><span class="nav-number">3.3.2.</span> <span class="nav-text">3.3.2 结果展示</span></a><ol class="nav-child"><li class="nav-item nav-level-4"><a class="nav-link" href="#Table-mode"><span class="nav-number">3.3.2.1.</span> <span class="nav-text">Table mode</span></a></li><li class="nav-item nav-level-4"><a class="nav-link" href="#Changlog-mode"><span class="nav-number">3.3.2.2.</span> <span class="nav-text">Changlog mode</span></a></li></ol></li><li class="nav-item nav-level-3"><a class="nav-link" href="#3-3-3-Environment-Files"><span class="nav-number">3.3.3.</span> <span class="nav-text">3.3.3 Environment Files</span></a></li></ol></li><li class="nav-item nav-level-2"><a class="nav-link" href="#3-4-Restful-API"><span class="nav-number">3.4.</span> <span class="nav-text">3.4 Restful API</span></a></li><li class="nav-item nav-level-2"><a class="nav-link" href="#3-5-Web"><span class="nav-number">3.5.</span> <span class="nav-text">3.5 Web</span></a></li></ol></li><li class="nav-item nav-level-1"><a class="nav-link" href="#4-结束"><span class="nav-number">4.</span> <span class="nav-text">4.结束</span></a></li></ol></div>
            

          </div>
        </div>
      <!--/noindex-->
      

      

    </div>
  </aside>
  


        
      </div>
    </main>

    <footer id="footer" class="footer">
      <div class="footer-inner">
        <div class="copyright">  <span class="exturl" data-url="aHR0cDovL3d3dy5iZWlhbi5taWl0Lmdvdi5jbg==">粤ICP备18110871号 </span>&copy; 2017 – <span itemprop="copyrightYear">2021</span>
  <span class="with-love" id="animate">
    <i class="fa fa-spinner"></i>
  </span>
  <span class="author" itemprop="copyrightHolder">dudefu</span>

  

  
</div>

<!--

  <div class="powered-by">由 <span class="exturl theme-link" data-url="aHR0cHM6Ly9oZXhvLmlv">Hexo</span> 强力驱动 v3.8.0</div>



  <span class="post-meta-divider">|</span>



  <div class="theme-info">主题 – <span class="exturl theme-link" data-url="aHR0cHM6Ly90aGVtZS1uZXh0Lm9yZw==">NexT.Pisces</span> v7.1.2</div>

-->



        








        
      </div>
    </footer>

    
      <div class="back-to-top">
        <i class="fa fa-arrow-up"></i>
        
          <span id="scrollpercent"><span>0</span>%</span>
        
      </div>
    

    

    

    
  </div>

  

<script>
  if (Object.prototype.toString.call(window.Promise) !== '[object Function]') {
    window.Promise = null;
  }
</script>














  
    
    
      
    
  
  <script color="26,26,26" opacity="0.5" zindex="-1" count="99" src="//cdn.jsdelivr.net/gh/theme-next/theme-next-canvas-nest@1/canvas-nest.min.js"></script>









  
  
  <script id="ribbon" size="300" alpha="0.6" zindex="-1" src="/lib/canvas-ribbon/canvas-ribbon.js"></script>



  



  
  <script src="/lib/jquery/index.js?v=3.4.1"></script>

  
  <script src="/lib/velocity/velocity.min.js?v=1.2.1"></script>

  
  <script src="/lib/velocity/velocity.ui.min.js?v=1.2.1"></script>

  
  <script src="/lib/reading_progress/reading_progress.js"></script>


  


  <script src="/js/utils.js?v=7.1.2"></script>

  <script src="/js/motion.js?v=7.1.2"></script>



  
  


  <script src="/js/affix.js?v=7.1.2"></script>

  <script src="/js/schemes/pisces.js?v=7.1.2"></script>



  
  <script src="/js/scrollspy.js?v=7.1.2"></script>
<script src="/js/post-details.js?v=7.1.2"></script>



  


  <script src="/js/next-boot.js?v=7.1.2"></script>


  
  <script src="/js/js.cookie.js?v=7.1.2"></script>
  <script src="/js/scroll-cookie.js?v=7.1.2"></script>


  
  <script src="/js/exturl.js?v=7.1.2"></script>


  

  
  

<script src="//cdn1.lncld.net/static/js/3.11.1/av-min.js"></script>



<script src="//unpkg.com/valine/dist/Valine.min.js"></script>

<script>
  var GUEST = ['nick', 'mail', 'link'];
  var guest = 'nick,mail,link';
  guest = guest.split(',').filter(function(item) {
    return GUEST.indexOf(item) > -1;
  });
  new Valine({
    el: '#comments',
    verify: true,
    notify: true,
    appId: '1N5rpk874DGudJw2wCL9J011-gzGzoHsz',
    appKey: '9Y83e6suJgx567wtxhKy45IN',
    placeholder: 'Just go go',
    avatar: 'mm',
    meta: guest,
    pageSize: '10' || 10,
    visitor: true,
    lang: 'zk-cn' || 'zh-cn'
  });
</script>




  
    <script>
  window.livereOptions = {
    refer: 'Apache Flink 零基础入门（四）：客户端操作的 5 种模式.html'
  };
  (function(d, s) {
    var j, e = d.getElementsByTagName(s)[0];
    if (typeof LivereTower === 'function') { return; }
    j = d.createElement(s);
    j.src = 'https://cdn-city.livere.com/js/embed.dist.js';
    j.async = true;
    e.parentNode.insertBefore(j, e);
  })(document, 'script');
</script>

  


  
  <script>
    // Popup Window;
    var isfetched = false;
    var isXml = true;
    // Search DB path;
    var search_path = "search.xml";
    if (search_path.length === 0) {
      search_path = "search.xml";
    } else if (/json$/i.test(search_path)) {
      isXml = false;
    }
    var path = "/" + search_path;
    // monitor main search box;

    var onPopupClose = function (e) {
      $('.popup').hide();
      $('#local-search-input').val('');
      $('.search-result-list').remove();
      $('#no-result').remove();
      $(".local-search-pop-overlay").remove();
      $('body').css('overflow', '');
    }

    function proceedsearch() {
      $("body")
        .append('<div class="search-popup-overlay local-search-pop-overlay"></div>')
        .css('overflow', 'hidden');
      $('.search-popup-overlay').click(onPopupClose);
      $('.popup').toggle();
      var $localSearchInput = $('#local-search-input');
      $localSearchInput.attr("autocapitalize", "none");
      $localSearchInput.attr("autocorrect", "off");
      $localSearchInput.focus();
    }

    // search function;
    var searchFunc = function(path, search_id, content_id) {
      'use strict';

      // start loading animation
      $("body")
        .append('<div class="search-popup-overlay local-search-pop-overlay">' +
          '<div id="search-loading-icon">' +
          '<i class="fa fa-spinner fa-pulse fa-5x fa-fw"></i>' +
          '</div>' +
          '</div>')
        .css('overflow', 'hidden');
      $("#search-loading-icon").css('margin', '20% auto 0 auto').css('text-align', 'center');

      

      $.ajax({
        url: path,
        dataType: isXml ? "xml" : "json",
        async: true,
        success: function(res) {
          // get the contents from search data
          isfetched = true;
          $('.popup').detach().appendTo('.header-inner');
          var datas = isXml ? $("entry", res).map(function() {
            return {
              title: $("title", this).text(),
              content: $("content",this).text(),
              url: $("url" , this).text()
            };
          }).get() : res;
          var input = document.getElementById(search_id);
          var resultContent = document.getElementById(content_id);
          var inputEventFunction = function() {
            var searchText = input.value.trim().toLowerCase();
            var keywords = searchText.split(/[\s\-]+/);
            if (keywords.length > 1) {
              keywords.push(searchText);
            }
            var resultItems = [];
            if (searchText.length > 0) {
              // perform local searching
              datas.forEach(function(data) {
                var isMatch = false;
                var hitCount = 0;
                var searchTextCount = 0;
                var title = data.title.trim();
                var titleInLowerCase = title.toLowerCase();
                var content = data.content.trim().replace(/<[^>]+>/g,"");
                
                var contentInLowerCase = content.toLowerCase();
                var articleUrl = decodeURIComponent(data.url).replace(/\/{2,}/g, '/');
                var indexOfTitle = [];
                var indexOfContent = [];
                // only match articles with not empty titles
                if(title != '') {
                  keywords.forEach(function(keyword) {
                    function getIndexByWord(word, text, caseSensitive) {
                      var wordLen = word.length;
                      if (wordLen === 0) {
                        return [];
                      }
                      var startPosition = 0, position = [], index = [];
                      if (!caseSensitive) {
                        text = text.toLowerCase();
                        word = word.toLowerCase();
                      }
                      while ((position = text.indexOf(word, startPosition)) > -1) {
                        index.push({position: position, word: word});
                        startPosition = position + wordLen;
                      }
                      return index;
                    }

                    indexOfTitle = indexOfTitle.concat(getIndexByWord(keyword, titleInLowerCase, false));
                    indexOfContent = indexOfContent.concat(getIndexByWord(keyword, contentInLowerCase, false));
                  });
                  if (indexOfTitle.length > 0 || indexOfContent.length > 0) {
                    isMatch = true;
                    hitCount = indexOfTitle.length + indexOfContent.length;
                  }
                }

                // show search results

                if (isMatch) {
                  // sort index by position of keyword

                  [indexOfTitle, indexOfContent].forEach(function (index) {
                    index.sort(function (itemLeft, itemRight) {
                      if (itemRight.position !== itemLeft.position) {
                        return itemRight.position - itemLeft.position;
                      } else {
                        return itemLeft.word.length - itemRight.word.length;
                      }
                    });
                  });

                  // merge hits into slices

                  function mergeIntoSlice(text, start, end, index) {
                    var item = index[index.length - 1];
                    var position = item.position;
                    var word = item.word;
                    var hits = [];
                    var searchTextCountInSlice = 0;
                    while (position + word.length <= end && index.length != 0) {
                      if (word === searchText) {
                        searchTextCountInSlice++;
                      }
                      hits.push({position: position, length: word.length});
                      var wordEnd = position + word.length;

                      // move to next position of hit

                      index.pop();
                      while (index.length != 0) {
                        item = index[index.length - 1];
                        position = item.position;
                        word = item.word;
                        if (wordEnd > position) {
                          index.pop();
                        } else {
                          break;
                        }
                      }
                    }
                    searchTextCount += searchTextCountInSlice;
                    return {
                      hits: hits,
                      start: start,
                      end: end,
                      searchTextCount: searchTextCountInSlice
                    };
                  }

                  var slicesOfTitle = [];
                  if (indexOfTitle.length != 0) {
                    slicesOfTitle.push(mergeIntoSlice(title, 0, title.length, indexOfTitle));
                  }

                  var slicesOfContent = [];
                  while (indexOfContent.length != 0) {
                    var item = indexOfContent[indexOfContent.length - 1];
                    var position = item.position;
                    var word = item.word;
                    // cut out 100 characters
                    var start = position - 20;
                    var end = position + 80;
                    if(start < 0){
                      start = 0;
                    }
                    if (end < position + word.length) {
                      end = position + word.length;
                    }
                    if(end > content.length){
                      end = content.length;
                    }
                    slicesOfContent.push(mergeIntoSlice(content, start, end, indexOfContent));
                  }

                  // sort slices in content by search text's count and hits' count

                  slicesOfContent.sort(function (sliceLeft, sliceRight) {
                    if (sliceLeft.searchTextCount !== sliceRight.searchTextCount) {
                      return sliceRight.searchTextCount - sliceLeft.searchTextCount;
                    } else if (sliceLeft.hits.length !== sliceRight.hits.length) {
                      return sliceRight.hits.length - sliceLeft.hits.length;
                    } else {
                      return sliceLeft.start - sliceRight.start;
                    }
                  });

                  // select top N slices in content

                  var upperBound = parseInt('1');
                  if (upperBound >= 0) {
                    slicesOfContent = slicesOfContent.slice(0, upperBound);
                  }

                  // highlight title and content

                  function highlightKeyword(text, slice) {
                    var result = '';
                    var prevEnd = slice.start;
                    slice.hits.forEach(function (hit) {
                      result += text.substring(prevEnd, hit.position);
                      var end = hit.position + hit.length;
                      result += '<b class="search-keyword">' + text.substring(hit.position, end) + '</b>';
                      prevEnd = end;
                    });
                    result += text.substring(prevEnd, slice.end);
                    return result;
                  }

                  var resultItem = '';

                  if (slicesOfTitle.length != 0) {
                    resultItem += "<li><a href='" + articleUrl + "' class='search-result-title'>" + highlightKeyword(title, slicesOfTitle[0]) + "</a>";
                  } else {
                    resultItem += "<li><a href='" + articleUrl + "' class='search-result-title'>" + title + "</a>";
                  }

                  slicesOfContent.forEach(function (slice) {
                    resultItem += "<a href='" + articleUrl + "'>" +
                      "<p class=\"search-result\">" + highlightKeyword(content, slice) +
                      "...</p>" + "</a>";
                  });

                  resultItem += "</li>";
                  resultItems.push({
                    item: resultItem,
                    searchTextCount: searchTextCount,
                    hitCount: hitCount,
                    id: resultItems.length
                  });
                }
              })
            };
            if (keywords.length === 1 && keywords[0] === "") {
              resultContent.innerHTML = '<div id="no-result"><i class="fa fa-search fa-5x"></i></div>'
            } else if (resultItems.length === 0) {
              resultContent.innerHTML = '<div id="no-result"><i class="fa fa-frown-o fa-5x"></i></div>'
            } else {
              resultItems.sort(function (resultLeft, resultRight) {
                if (resultLeft.searchTextCount !== resultRight.searchTextCount) {
                  return resultRight.searchTextCount - resultLeft.searchTextCount;
                } else if (resultLeft.hitCount !== resultRight.hitCount) {
                  return resultRight.hitCount - resultLeft.hitCount;
                } else {
                  return resultRight.id - resultLeft.id;
                }
              });
              var searchResultList = '<ul class=\"search-result-list\">';
              resultItems.forEach(function (result) {
                searchResultList += result.item;
              })
              searchResultList += "</ul>";
              resultContent.innerHTML = searchResultList;
            }
          }

          if ('auto' === 'auto') {
            input.addEventListener('input', inputEventFunction);
          } else {
            $('.search-icon').click(inputEventFunction);
            input.addEventListener('keypress', function (event) {
              if (event.keyCode === 13) {
                inputEventFunction();
              }
            });
          }

          // remove loading animation
          $(".local-search-pop-overlay").remove();
          $('body').css('overflow', '');

          proceedsearch();
        }
      });
    }

    // handle and trigger popup window;
    $('.popup-trigger').click(function(e) {
      e.stopPropagation();
      if (isfetched === false) {
        searchFunc(path, 'local-search-input', 'local-search-result');
      } else {
        proceedsearch();
      };
    });

    $('.popup-btn-close').click(onPopupClose);
    $('.popup').click(function(e){
      e.stopPropagation();
    });
    $(document).on('keyup', function (event) {
      var shouldDismissSearchPopup = event.which === 27 &&
        $('.search-popup').is(':visible');
      if (shouldDismissSearchPopup) {
        onPopupClose();
      }
    });
  </script>





  

  
  <script src="https://www.gstatic.com/firebasejs/4.6.0/firebase.js"></script>
  <script src="https://www.gstatic.com/firebasejs/4.6.0/firebase-firestore.js"></script>
  
    <script src="https://cdnjs.cloudflare.com/ajax/libs/bluebird/3.5.1/bluebird.core.min.js"></script>
  
  <script>
    (function () {

      firebase.initializeApp({
        apiKey: '',
        projectId: ''
      })

      function getCount(doc, increaseCount) {
        //increaseCount will be false when not in article page

        return doc.get().then(function (d) {
          var count
          if (!d.exists) { //has no data, initialize count
            if (increaseCount) {
              doc.set({
                count: 1
              })
              count = 1
            }
            else {
              count = 0
            }
          }
          else { //has data
            count = d.data().count
            if (increaseCount) {
              if (!(window.localStorage && window.localStorage.getItem(title))) { //if first view this article
                doc.set({ //increase count
                  count: count + 1
                })
                count++
              }
            }
          }
          if (window.localStorage && increaseCount) { //mark as visited
            localStorage.setItem(title, true)
          }

          return count
        })
      }

      function appendCountTo(el) {
        return function (count) {
          $(el).append(
            $('<span>').addClass('post-visitors-count').append(
              $('<span>').addClass('post-meta-divider').text('|')
            ).append(
              $('<span>').addClass('post-meta-item-icon').append(
                $('<i>').addClass('fa fa-users')
              )
              ).append($('<span>').text('阅读次数 ' + count))
          )
        }
      }

      var db = firebase.firestore()
      var articles = db.collection('articles')

      //https://hexo.io/docs/variables.html
      var isPost = 'Apache Flink 零基础入门（四）：客户端操作的 5 种模式'.length > 0
      var isArchive = '' === 'true'
      var isCategory = ''.length > 0
      var isTag = ''.length > 0

      if (isPost) { //is article page
        var title = 'Apache Flink 零基础入门（四）：客户端操作的 5 种模式'
        var doc = articles.doc(title)

        getCount(doc, true).then(appendCountTo($('.post-meta')))
      }
      else if (!isArchive && !isCategory && !isTag) { //is index page
        var titles = [] //array to titles

        var postsstr = '' //if you have a better way to get titles of posts, please change it
        eval(postsstr)

        var promises = titles.map(function (title) {
          return articles.doc(title)
        }).map(function (doc) {
          return getCount(doc)
        })
        Promise.all(promises).then(function (counts) {
          var metas = $('.post-meta')
          counts.forEach(function (val, idx) {
            appendCountTo(metas[idx])(val)
          })
        })
      }
    })()
  </script>


  
  

  
  

  


  
<script>
if ($('body').find('div.pdf').length) {
  $.ajax({
    type: 'GET',
    url: '//cdn.jsdelivr.net/npm/pdfobject@2/pdfobject.min.js',
    dataType: 'script',
    cache: true,
    success: function() {
      $('body').find('div.pdf').each(function(i, o) {
        PDFObject.embed($(o).attr('target'), $(o), {
          pdfOpenParams: {
            navpanes: 0,
            toolbar: 0,
            statusbar: 0,
            pagemode: 'thumbs',
            view: 'FitH'
          },
          PDFJS_URL: '/lib/pdf/web/viewer.html',
          height: $(o).attr('height') || '500px'
        });
      });
    },
  });
}
</script>


  
<script>
if ($('body').find('pre.mermaid').length) {
  $.ajax({
    type: 'GET',
    url: '//cdn.jsdelivr.net/npm/mermaid@8/dist/mermaid.min.js',
    dataType: 'script',
    cache: true,
    success: function() {
      mermaid.initialize({
        theme: 'dark',
        logLevel: 3,
        flowchart: { curve: 'linear' },
        gantt: { axisFormat: '%m/%d/%Y' },
        sequence: { actorMargin: 50 }
      });
    }
  });
}
</script>


  
  <script>
    (function(){
      var bp = document.createElement('script');
      var curProtocol = window.location.protocol.split(':')[0];
      bp.src = (curProtocol === 'https') ? 'https://zz.bdstatic.com/linksubmit/push.js' : 'http://push.zhanzhang.baidu.com/push.js';
      var s = document.getElementsByTagName("script")[0];
      s.parentNode.insertBefore(bp, s);
    })();
  </script>


  

  

  

  

  
  
  
  <script src="/lib/bookmark/bookmark.min.js?v=1.0"></script>
  <script>
  
    bookmark.scrollToMark('auto', "#更多");
  
  </script>


  
<script>
  $('.highlight').not('.gist .highlight').each(function(i, e) {
    var $wrap = $('<div>').addClass('highlight-wrap');
    $(e).after($wrap);
    $wrap.append($('<button>').addClass('copy-btn').append('复制').on('click', function(e) {
      var code = $(this).parent().find('.code').find('.line').map(function(i, e) {
        return $(e).text();
      }).toArray().join('\n');
      var ta = document.createElement('textarea');
      var yPosition = window.pageYOffset || document.documentElement.scrollTop;
      ta.style.top = yPosition + 'px'; // Prevent page scroll
      ta.style.position = 'absolute';
      ta.style.opacity = '0';
      ta.readOnly = true;
      ta.value = code;
      document.body.appendChild(ta);
      const selection = document.getSelection();
      const selected = selection.rangeCount > 0 ? selection.getRangeAt(0) : false;
      ta.select();
      ta.setSelectionRange(0, code.length);
      ta.readOnly = false;
      var result = document.execCommand('copy');
      
        if (result) $(this).text('复制成功');
        else $(this).text('复制失败');
      
      ta.blur(); // For iOS
      $(this).blur();
      if (selected) {
        selection.removeAllRanges();
        selection.addRange(selected);
      }
    })).on('mouseleave', function(e) {
      var $b = $(this).find('.copy-btn');
      setTimeout(function() {
        $b.text('复制');
      }, 300);
    }).append(e);
  })
</script>


  

  

</body>
</html>
